{"count":4769,"measurements":[{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e38913cbde771d7d815f54d2bc1ae96b7862b0851b259c413230d11b47f0ce37","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-09-22"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Opus 5.5; model release: 2026-09-22","source_accessibility":"API access","source_model_name":"Claude Opus 5.5","source_model_release_date":"2026-09-22","source_model_versions":["claude-opus-5-5_max","claude-opus-5-5_medium","claude-opus-5-5_xhigh"],"source_reasoning_efforts":["medium","xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5a43b558192ff577a0b1e28a6d61daa1a17a3dd6975687615119b102ef8e5ddb","result":{"confidence_high":172.0,"confidence_low":164.0,"sample_count":null,"score":167.35,"score_display":"167.35","source_stated_rank":1},"run_fingerprint":"2c529c9ea0a2d9ab5dbf8706f790a59543cdf2cc3c32ac61f41689a814754193","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f9bccce8d4fa3cdf550e517a","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ba0ab42805522f0b6a92109fc4654eb75aeec80b8ef552045fc042ea5dabb97b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"6298f50951000333523fcffa04743bb462e1d1c5dc188ba709b7e91e27eeaa75","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.62237,"score_display":"57.6","source_stated_rank":null},"run_fingerprint":"38ba43918cc4a79e2369d609cc6a1764f4e8042caf1c25025f2acd95230c503d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b643394a1d8c67a8e06018e38bac70427f7b184f55302698427b5b0325ee24c8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-xhigh","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"6c6706cf3eb058774c129791d1c3ced3e5ca5209f895d0afd6f11a228d1f59e2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.987351,"score_display":"56.0","source_stated_rank":null},"run_fingerprint":"1010c7172fa8e9bb265622bba06bbea50decbe505d5baa88210d67d2902f93fd","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"44c38ce13864102f7fe63a883e8f7635990282864a364dffd8846d7480132a44","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-high","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"bc5fed6a8da1984b31b750354dbb8c01799634307d2ab6c2afce96e1c80ddf8b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.583196,"score_display":"53.6","source_stated_rank":null},"run_fingerprint":"5012bcb1b10e7447eb2b8ee26597662d591f494de3aeb829b25fd7d261728757","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e732832b317de5fa5b4887baa81cb1e9994b4b80923080b88eab05e730c1ccaf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-medium","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"e8ad0dbcecbcb4badb9fe9e2d158d7618053d21756cd8e911b96fd8b5e51029b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.243493,"score_display":"51.2","source_stated_rank":null},"run_fingerprint":"58b8f499a5da6c0cdcab8b4291c13fd7cb09363bc4665bac85525d4e11170309","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4f4bb4bfdd2bd39db1cf8d63522653e996ecb849d0d480d3da3fb8d71e6e5851","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-low","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"0deffc5cfb8857ac94b993e09238bf98e94ac6e7d57a8a290c1a6d0202aaa34c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.307778,"score_display":"42.3","source_stated_rank":null},"run_fingerprint":"3654c88900ff6949518d4445396ea7e64f6ee86be51f4b9e065ec272ae0385ef","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9596c22ea907452d8017c005ce455d49bc61c9bf38856ba1661c7645298b54bf","metric_details":{"license":"Proprietary","variance":21.397610384076472},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1512.7618814402601,"confidence_low":1494.629254586802,"sample_count":4552,"score":1503.695568013531,"score_display":"1504","source_stated_rank":4},"run_fingerprint":"da672251815987a9ff2b7ced2a35ac846e96baa98d7d17c6b88630399f601cec","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8ed3f237302c9fabf96a2100273e3570afcffdbfc76ccf29d36d2628a30e826d","metric_details":{"category_scores":{"Agentic Coding":71.717,"Coding":89.253,"Data Analysis":80.307,"IF":65.73775,"Language":86.27,"Mathematics":97.07749999999999,"Reasoning":92.15375}},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.21657142857143,"score_display":"83.22","source_stated_rank":2},"run_fingerprint":"0795e9225dd107421be42f85d47a4708f0d44c7fedaa6eaab9a0e9fac2320ca6","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"460c0cd01c0e12b9d349224888ebec06c117fa0943cccff26247ba8800799d8d","metric_details":{"category_scores":{"Agentic Coding":65.35333333333334,"Coding":89.253,"Data Analysis":79.803,"IF":67.04975,"Language":85.49866666666667,"Mathematics":96.803,"Reasoning":90.65375}},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.05921428571428,"score_display":"82.06","source_stated_rank":5},"run_fingerprint":"c4ddef52f99c07bbe9b81842c4fc3cfe15176ce17df67c81bbf621576c907382","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"91413c18d4e97ea545d8b9872a50378910fe1394c1e2a4ffbf097f76c41aba77","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1c596ce98a0be16ef505f3b3465287c9e28f71e51ce6d58f780db55383547a50","result":{"confidence_high":63.385702,"confidence_low":59.280159,"sample_count":2158,"score":61.353105,"score_display":"61.4","source_stated_rank":null},"run_fingerprint":"7ddb4b5a1e61611d5d29218f32673ad48ee5ba5e701c2f25176c1f033ff745db","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d5da2150b1b06ef8790bb0ff75eabc410d10e4c9fa1ccc3b4b6fcd19bf9959ab","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-xhigh","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"3cd15e9ae964ad6bac92c962b903b54e495928564e26976f58fc529f13209c03","result":{"confidence_high":59.577488,"confidence_low":55.409734,"sample_count":2158,"score":57.506951,"score_display":"57.5","source_stated_rank":null},"run_fingerprint":"10fbcd9dfa02f61995232962795859178c3cc2691cb3347a5dad9120d95efd61","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"58e4742820b831ccf64adedda417d67324cb6008c791e4006754a3cdfd6e1ff8","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-high","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b0a4672d0333faf415e7ec2718e594f0ec1aeb6bd497485fae6d2d384f9afe30","result":{"confidence_high":57.645497,"confidence_low":53.456149,"sample_count":2158,"score":55.560704,"score_display":"55.6","source_stated_rank":null},"run_fingerprint":"bb8275585a29ee804fce21d0734f76a856f0b0fc3971124cb562eca5efc21f53","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9c2547be90cecbebd91236f331443920a66960a233ec2a87e52234404b629ceb","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-medium","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"93945e5b82778549cdd3c63e64fd9904bcd621c8b0abad49f0a22453df4586f3","result":{"confidence_high":56.816504,"confidence_low":52.619896,"sample_count":2158,"score":54.726599,"score_display":"54.7","source_stated_rank":null},"run_fingerprint":"564534a3ca650542417d17de9c251237c26f6d28908c41153899fbbdefa9e156","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5902749943c4f8f996017abe391c13d0cd61917e4f463c4fd98297dafaf55ba8","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-low","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"64bf4d1340716d5c9d062d128634f545416faa8afea43e296e8ff1a99f20c110","result":{"confidence_high":50.441308,"confidence_low":46.228199,"sample_count":2158,"score":48.331789,"score_display":"48.3","source_stated_rank":null},"run_fingerprint":"e203860095c1e8a7f9262c08af160779fa8ae5a79266d24fc19f8a8652825f49","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T18:27:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T18:27:39.000Z","first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7d83cff392888bd5138e6b41c95c150c81b59ce9f3f61b8ab304f39beebbce14","metric_details":{"best_score_across_scorers":"0.722","model_release_date":"2026-09-22","stderr":1.4174516461485274},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CgLfiGcNwMPVzxr8GdNUAK","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"63ee90334824e9cf59ed152b9627654b2fcb965eef7037ccf787c24b853603fe","result":{"confidence_high":74.97820522645111,"confidence_low":69.4217947735489,"sample_count":1000,"score":72.2,"score_display":"72.2","source_stated_rank":4},"run_fingerprint":"5128522de5af9447c8076a8efcec049ead359c8d9b9d63d78c014775c88a1e00","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":12,"at":"2026-09-22T18:27:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T18:27:39.000Z","first_observed_at":"2026-09-30T18:53:22Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b36dffed209a00b2245e9ce5a3761c20d736265ca6a838553d1f9215a6b35e6b","metric_details":{"best_score_across_scorers":"0.9059343434343434","model_release_date":"2026-09-22","stderr":1.9434397190192136},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6gmbahPSxHf9xVhAYqpnhi","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"919ebf8a3d132d097dbb0cc21c7ba269afb98b98144deb645c0be6e6390220de","result":{"confidence_high":94.402576192712,"confidence_low":86.78429249415667,"sample_count":null,"score":90.59343434343434,"score_display":"90.6","source_stated_rank":40},"run_fingerprint":"73e708cdd6e6ea56029fdc403564aef67b761ee3d3b566e32499614f33987885","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T18:27:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T18:27:39.000Z","first_observed_at":"2026-09-30T18:53:12Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"70b5ebca22043c799377af32c88522c5f9a2780e7c2c5f29ab694994c0f62904","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-09-22","stderr":0.0},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LSfona7P3jCAnZFXJ2YA7F","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4ab8432dffdd095ab10181addaf3fcc8c8f586ceccecd8dbf74f176305c5c768","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"d235767b09bba93802a48e36381b76dc0e28e35ff6889e5b4dec8ece25eabe2a","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T18:27:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T18:27:39.000Z","first_observed_at":"2026-09-30T18:53:13Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7106cb5c83e4df8599eaafb9c5ba7662429645c9688604e88c1eb3ae90241335","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.9122807017543859","leaderboard_private_problem_count":285,"model_release_date":"2026-09-22","public_example_count":10,"stderr":1.6786209858442096},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"LPxhvBNEn7S3xkRZUgcvc9","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 5.5 (max)","thinking":null,"upstream_model_id":"claude-opus-5-5_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"4e2bced60f1ef278854d63ac4e8ca31823c2c92c09c92122c32905e6348ade6f","result":{"confidence_high":94.51816730769323,"confidence_low":87.93797304318394,"sample_count":285,"score":91.22807017543859,"score_display":"91.2","source_stated_rank":3},"run_fingerprint":"8b0546585a59680027e094dd1146cd522b1112dd49059eb5fa2896b47a32812a","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2eb7052dfaec5ca80229f4bcd3cf54f01e888d5e5f7ca1f036b42ffcf137d30c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.8517409666666667,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-opus-5-5"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-opus-5-5","model_type":"CoT","upstream_model_id":"anthropic-claude-opus-5-5-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"315b4f3d86330f1b0ad624dbc9cb71b7d38a32a3451c9f98622199f6464ab2d3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.66666666666666,"score_display":"92","source_stated_rank":10},"run_fingerprint":"87bff8a27e7ec7fc0172bbef3c92bdc60ea4621be6f313e6674a58758d21b2eb","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9b0abf956b83f3fac8a5587ff62b139449afde723322176ad9f377247d6ca3a0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.6709066333333333,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-opus-5-5"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-opus-5-5","model_type":"CoT","upstream_model_id":"anthropic-claude-opus-5-5-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1d6ccc4a65daa770af4494c4448330a5c79b167199257e2d5a73734c59a04240","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.5,"score_display":"92","source_stated_rank":6},"run_fingerprint":"839b98c189bfe3dea4c3756d86623ab64070b4dcabad643721b61cd7586fc553","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"078ff4657772d0fbfe66cd6124fb9a9fef4a2831d0f9bbbfc0ca1530b73a8c81","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.4078964666666666,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-opus-5-5"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-opus-5-5","model_type":"CoT","upstream_model_id":"anthropic-claude-opus-5-5-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8a744ce23d3152bf3870ec4a1f06637c05a625842d6d789b9aa0d553ca88ed62","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.33333333333333,"score_display":"93","source_stated_rank":4},"run_fingerprint":"675f5edb13c5e86eba99e2214113f47ab2de88770d8dc2146bfc4e3ce75faa02","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"044ae11349df3e4884fd42898b877c7e38b0181690094e6df494c353047082cd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.3369188,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-opus-5-5"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-opus-5-5","model_type":"CoT","upstream_model_id":"anthropic-claude-opus-5-5-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bedf52cda78ffb57ee231b574af91c008418a9e3b6d7455d138ecd501ef054d4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.5,"score_display":"88","source_stated_rank":23},"run_fingerprint":"327d76721ed212cf0c3b1ee209b0296e422d210d606fa4ec11f7f18ae4c30f01","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"af4fa4eeacd9d00682fb052ce58a89241ce7257b0748c127832be105394c969a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.24127179999999998,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-opus-5-5"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-opus-5-5","model_type":"CoT","upstream_model_id":"anthropic-claude-opus-5-5-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"770ce1fe03b8a8cfa7e1e31b49d263842d94605c0fbb98a8f4595233fba42e27","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.1388888888889,"score_display":"70","source_stated_rank":54},"run_fingerprint":"2c5fa3d78ff5e274ba270edd257482e540862b672535ef479468bd8d67330bc1","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ad2dc6a7cd0fa77b6b5fda5e1a0591bb59a7f6426a3ebdbe2d750e4f6d6e6ca9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Max)","source_effort":null,"source_model_version":"claude-opus-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"fc51ee96d5b83d1fcce94b450100fce80f4a093619554e67d6e0e15e1c49e134","result":{"confidence_high":94.100582,"confidence_low":88.352881,"sample_count":360,"score":91.66666666666666,"score_display":"91.7","source_stated_rank":9},"run_fingerprint":"0c3e8a98855043bb46d955eb05bcfdc8f0f27bbf3ded7f039d95a3ea9e9cb5dc","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b82d97845ec9cce72a675a3ac28c358e4cd44639f4f32e2df1f5f6ba91284d92","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (XHigh)","source_effort":null,"source_model_version":"claude-opus-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f91922c2b1e15bb7b895576c17a95b745c21b74e93cf1c31327e1581ab3b8c48","result":{"confidence_high":94.794673,"confidence_low":89.30786,"sample_count":360,"score":92.5,"score_display":"92.5","source_stated_rank":5},"run_fingerprint":"bc86c535c1ad5de0e21a8c4eaf6e88e6e7173b1415dfcd4e721efdcdde890c9e","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e57ea70c26e4c34c17b35c1983414758b7ff62ef57dec162f5bb7ee49449858d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (High)","source_effort":null,"source_model_version":"claude-opus-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"be807826111a9698ff750c861eb5540ab7659c142955c67a5bc1fb0ba7b134a9","result":{"confidence_high":95.479458,"confidence_low":90.272143,"sample_count":360,"score":93.33333333333333,"score_display":"93.3","source_stated_rank":3},"run_fingerprint":"7b6135c5ac94a4fe53aac2c3403afb7bc8cd5dcba0ef0f38082dd38339933570","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6cc3ec7d190d6953d145e27cd5b9be6bf7c8b0b80d13b680f951533aa19513bd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Medium)","source_effort":null,"source_model_version":"claude-opus-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d1f669974908ae615a5bfdde2ca3d5bdae7e802554417d7d8e6f0bcd630795e0","result":{"confidence_high":90.525325,"confidence_low":83.682792,"sample_count":360,"score":87.5,"score_display":"87.5","source_stated_rank":23},"run_fingerprint":"396cf8e31af42764e439ff48f6151c8117f61e772ed3c9b5128c38cc03d5b039","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0929464ff0b831f964b114a1b098ba82c3836d549659597359c974759788c770","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Low)","source_effort":null,"source_model_version":"claude-opus-5-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a28d145da26219d8a945708af446f34a535216d546915fb59096a52722f74fe1","result":{"confidence_high":74.633597,"confidence_low":65.218911,"sample_count":360,"score":70.1388888888889,"score_display":"70.1","source_stated_rank":57},"run_fingerprint":"032b1894c930936a5ae5c146740f7c5d557c23f9762b016eab4fdcc0b1a4f3e6","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T18:27:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T18:27:39.000Z","first_observed_at":"2026-09-30T18:53:13Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d443cc9b90b24fd57df998e13e385e0d67fa519c2d88b966fa785fd4a0ec8a76","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.95","leaderboard_private_problem_count":41,"model_release_date":"2026-09-22","public_example_count":2,"stderr":4.117547077105885},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"9qKrTxPhpfW985gieKU56x","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 5.5 (max)","thinking":null,"upstream_model_id":"claude-opus-5-5_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"f0f19eb269aa45592a3427a4bb852adbd365864a3ad88b0c430a42d42ef2d8f8","result":{"confidence_high":100.0,"confidence_low":86.92960772887247,"sample_count":41,"score":95.0,"score_display":"95.0","source_stated_rank":5},"run_fingerprint":"e649362b392f4f2bb1c29ac7a8581162e05e1184f2b029cc1e18eabaf9cefa5f","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a4756f63e2c0631fa5b05bf244c398ad3317aa7238d5d1550e312162c3677170","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"961f86fd7cf087a1e0f1cc2d5ea011c42695432a68fb76dacc7b1b34604366c6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.687861,"score_display":"87.7","source_stated_rank":null},"run_fingerprint":"f643047e1d5a6ad5e93be137c70fb19ce478776fe6d1dae12a6ae91d6d692ae3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e56a5b793f27ac159342b9d14bfb7ae84040179852200eb818911e1a146019e5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-xhigh","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"503d17f803041fb534545f189cb3f81fb81b6e5bfe0115d5fba23365a68c3bfa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.589595,"score_display":"86.6","source_stated_rank":null},"run_fingerprint":"47dd09f7e7a03c6af2c527762f778b1cb28f8eb3c277d4952b63c2fc106eb4ff","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c841d4aabdea687b32ffb2536745392b5de86ab67689e3e03aced0bf1be0f12b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-high","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"3dc29931bc87bbc0d81a206046542a3f41c73d5bb4a89d6cdd78d66d7144991c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.83815,"score_display":"85.8","source_stated_rank":null},"run_fingerprint":"ff14d1606d1a6ef04e4e07f6e11f26e8c49a77e311910a2cd1367b15973d91a7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cdcacfce883af334bbabddfe299948ebd0a9d2161e7d1b228c319d82580e3011","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-medium","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"11b33a33b6981f8f8e5d8ccc91118e2489012bc6d16c15dc86c8fa275f825509","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.722543,"score_display":"85.7","source_stated_rank":null},"run_fingerprint":"5687ecbb1bd1c9be9726b0defe26a617c11a744c0986918e0d8eaf1df7010272","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"afe046df0b81ed7e149fef793eae315731a75beebb5914dba7161dde009d0db0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-low","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f8c655e469ac0e6645dde39f187229032f2f6dec99324a58f13630fd98dfeb63","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.682081,"score_display":"84.7","source_stated_rank":null},"run_fingerprint":"74932614559e01934381f78256f9f374fbfd2dddf218aa6483c9a9e19f6ff693","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"97fd7a50bc8a4b1f3efe4cecd6d465e6a0ef179caa3b4a950a60f34aa828c680","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"76a97c629ae6d69ef9e2d4029a7638c0052a0e86f8643667b1c6f9f10ce0b974","result":{"confidence_high":43.237892,"confidence_low":22.067881,"sample_count":71,"score":31.7142857142857,"score_display":"31.7","source_stated_rank":2},"run_fingerprint":"965d03b8fea4aa26dc73a7d35c47375438ec91718d1cec4c197a5c24c376166d","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5cdb62d6e63fb75e64a32cf49b10c6180cd81a338aee58ec8b7e489f7889b38d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"f73b3893f7fed8b960c7c04f5a212bae9e070024dbff19521a0ddb502da83243","result":{"confidence_high":43.237892,"confidence_low":22.067881,"sample_count":71,"score":31.7142857142857,"score_display":"31.7","source_stated_rank":2},"run_fingerprint":"6562eb3c8ff1b22f93a97155a84fed0731123a8fa018cfcde3e8ed185700e7fc","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e0f39a0a57cea7c92c31afd67c03f741cebe3805d93176b3ae1a76ff68a5bc73","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (High, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"2a253b490dd4c2d508cad6b4df41665bf264a672b42b9f15b784a0252c820ca0","result":{"confidence_high":42.350693,"confidence_low":21.328788,"sample_count":71,"score":30.8571428571429,"score_display":"30.9","source_stated_rank":11},"run_fingerprint":"fb0c482cd16bc9c701a0a658ec185bd4c011fdbcc7e8f1868d099a44c23fc83a","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"76daeb12f9e82dac6fb1b8a3102640afaebaf7e1b5497e02cded0c8ce69dc8c7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Medium, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"34b899b6e3fd1a290ed9b07e812d178c36de9a615b17e3cadad6b8d4d2e6fe1e","result":{"confidence_high":39.063098,"confidence_low":18.653313,"sample_count":71,"score":27.7142857142857,"score_display":"27.7","source_stated_rank":27},"run_fingerprint":"f7e2e6a33d1f729ca069fefab9cc93c6796a2da105a0b93cf11cca5ac3d94c88","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"119b321a78905fcb4334e53099e64258782bb74e21ca14228a56c07ed34c9a55","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Low, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"e36bb4c7f24e337305ba1f2cca993652fbb326f9e2b88ca93bb3a173211a7b04","result":{"confidence_high":28.178675,"confidence_low":10.564331,"sample_count":71,"score":17.714285714285698,"score_display":"17.7","source_stated_rank":61},"run_fingerprint":"7f99164c3cac03fd2d95be76b9ff363f696e5ddb82fc1b80d1899127e9b40ea4","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d964d62172d74bfa1ea4661a8934adc0e3421d1b014fbe723bdd69393539789e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6b79faf5b141f13bb8519cc6bf910e017223e96f47f1afa5b73ce8ebfb110515","result":{"confidence_high":43.322603,"confidence_low":22.008593,"sample_count":70,"score":31.714286,"score_display":"31.7","source_stated_rank":null},"run_fingerprint":"397f7c3f7c4d75a56c038951eccb2ae791a3ff3f3d0bd63557d65568ee87bff6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"98c2fde17479edc7cae07dd6d407e78cd863a788c24b3ec57d35dd375cf2d522","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-xhigh","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"fbad37ae9ead4c844121f4e5a72284eee1de0a9173fba81269ede84249573893","result":{"confidence_high":43.322603,"confidence_low":22.008593,"sample_count":70,"score":31.714286,"score_display":"31.7","source_stated_rank":null},"run_fingerprint":"1dc8368ae6e87a42fdab05aacf7f3b5253cbd89acc47c303aace93cad7282e28","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d992acc5f64619a5d49149c3547b509082d67290e8d5a59eb42f2dfab4a3a392","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-high","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"e3926abd0d47430f8e55818034655870f05c00257f910a2325f7d2fd31e6ee0e","result":{"confidence_high":42.435559,"confidence_low":21.270536,"sample_count":70,"score":30.857143,"score_display":"30.9","source_stated_rank":null},"run_fingerprint":"ca72caaba1646c27f69f2f04bddfceb0a27b21b0869ffaa2b8a493bd0ec3c937","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"57211f64e7703013b21a4748f6de0778d0175ac538488bbd4fa051f65afa1180","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-medium","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5b8c9a595541ab4ae341eb391a368dba49deabdab17387edc18880fc5fd1e299","result":{"confidence_high":39.148332,"confidence_low":18.599063,"sample_count":70,"score":27.714286,"score_display":"27.7","source_stated_rank":null},"run_fingerprint":"fa725874a75c27989771e11ac148dc3772e8c4dfb25785f002bab320ab8de6c5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2d181dbb3373c29cb7462000fde247a845fa6e9dfed8ee701a01bb90d9c6c8c2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-low","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b2dd1adf38021a642dc3b31acb0f84c1d541f80a2a679a0ca87534434597d555","result":{"confidence_high":28.262679,"confidence_low":10.525214,"sample_count":70,"score":17.714286,"score_display":"17.7","source_stated_rank":null},"run_fingerprint":"6887ec40e966f8f65277324cbe93c7f961214d62d0140e9bc8fa5340a6e6b792","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2ca73b3dbaf14324f3bac94ee6e7faebb4631f65e1f4b5ac974b7b22390c978d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Max)","source_effort":null,"source_model_version":"claude-opus-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"55c1379708f54011fe6209562f23097353ba4c1673e14d76e4b935194212a208","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"54382c6ec463e68fae9d007c1896a965fce460ae6c6ba0b1ce93bb531d6024e2","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9cc63674c30f7de28f8055847b83e75f4f464aee76d64d80a59050658ad78efc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (XHigh)","source_effort":null,"source_model_version":"claude-opus-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"37353185c0c83ef3016caab0a15e1fb041ac141a24f2af4fea9dd0ed1de4735e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"ba349021329fb8dd5182d00961d9912055dd797880115f0535dec866353e6dd0","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"92e0fa56ccf8fce3e68bde6faabdbbe20cabadb52c2dfd8ffec0ab640baaeb1b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (High)","source_effort":null,"source_model_version":"claude-opus-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"1e7b84bd718b477a3d75162647a83fe849ed23c56b8d4a35926238421dd68bbb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":98.5,"score_display":"98.5","source_stated_rank":1},"run_fingerprint":"df4eadcd7e7031f0008ed3616696f623694a888465f037654c7260a61f359a0c","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"79db8de4034ca1ab670a2fbf0436a47c795194a49803ce134110b898cda81d18","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Medium)","source_effort":null,"source_model_version":"claude-opus-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6609d85563a722cac45a3fe8477cea2a7b878226fc535539e0290b453778a42b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"cc239de768bc5727e72783a9959ee404c0166348b56ad3463577b5d1459c256e","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"316b7a448a19ddbc3df9c127151edb2a60b10c89f4e9e7f4550435208ca7989b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Low)","source_effort":null,"source_model_version":"claude-opus-5-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"7e5adda7df823ef7729bc7080e7edb866ff9f9796f9d2df91c2f417b5d5c3bf1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.5,"score_display":"88.5","source_stated_rank":79},"run_fingerprint":"5e0f0c3579ef0f1df38e93144e048e3466eca605199123e22b8528e7264000e8","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1820579c2421c8ccdc277984d6eb74fe8a7fcfd6864a57b2c2a4d4a75f9505e6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"13.43","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"steps_per_task":"185","tokens_per_task":"218363","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5.5 Max","source_effort":"Max","source_model_version":"claude-opus-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"8ae70adb0c419622defa779bf614eb2048530c35c69a7400e6a7cb4291705178","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.8,"score_display":"57.8","source_stated_rank":1},"run_fingerprint":"7aa42158ed07dd233f9084c111e4d35b560892ca613334ae88364074eda32354","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0fe110eece561d82a6863503da75aaf814595495f8f1fd71e7a7602935b5eadb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"6.98","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"steps_per_task":"109","tokens_per_task":"101083","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5.5 Extra High","source_effort":"Extra High","source_model_version":"claude-opus-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"8ceecc5fcdab06f6f71287a795c53fe2d0dbf9b2a870f9a8abbf4a7a14cf7614","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.00000000000001,"score_display":"56.0","source_stated_rank":2},"run_fingerprint":"300477d08649e8b03065622daab3f1b2367576dd1cafa3bde3c5e210a3d0b6b6","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:45Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"116bb518bb64e88f071492ac1f29a93c26d553c71e94ec93cdc43919acd46ba3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"3.97","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"steps_per_task":"68","tokens_per_task":"53078","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5.5 High","source_effort":"High","source_model_version":"claude-opus-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"1cb810aa8bba452437ea6de8149433638e16e81d1f67207c52324a7b992a839d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.00000000000001,"score_display":"56.0","source_stated_rank":2},"run_fingerprint":"9e991c057cc6decdc9088c3483fec57d5b3c70b3e7f2e68fa660196659d9b59c","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:45Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c81f368632d73658a7cfd25803a7460bf90319dc5e922c9ce38285c20c04cd67","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.91","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"steps_per_task":"54","tokens_per_task":"37954","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5.5 Medium","source_effort":"Medium","source_model_version":"claude-opus-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"f330454d0d215f89fe191479acf9bbf8845bb8f37a9c787b56a2efb471dd1c99","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.5,"score_display":"52.5","source_stated_rank":6},"run_fingerprint":"dc36a3bdcafe5048050d2f266df890a6e3ac6113bba39c2a28b167dc783dd697","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:45Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b4d6fcdebbed63e5e5f454c38d2548b6f4d6df85780d84b5812d7b57317be487","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.17","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"steps_per_task":"28","tokens_per_task":"15811","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5.5 Low","source_effort":"Low","source_model_version":"claude-opus-5-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"fc67f0b69c3161b9ac660f044061e2a687c1738e8527f99ece75f7d007abbabe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.7,"score_display":"43.7","source_stated_rank":18},"run_fingerprint":"9f56d7edcdf187f9aa8d1f4d003ce69a62950c6f05b0afc2593a83e7d49b27ae","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4d1aba01ff7c872b37bfe7677fa744487b467cccc672c09724a56a1bb607513a","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5","source_effort":"medium","source_model_version":"claude-opus-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"f47b5fcf31c037c3984dae6d00e33dd6a78c5acf154abd1130cabe14bfdfd4c4","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":54.64,"score_display":"54.6","source_stated_rank":1},"run_fingerprint":"0c5d9afaaa61c56a8d8684cb6976df1b060c0ff66349ba95b151ed4d89cb853c","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":13,"at":"2026-09-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-09-22","status":"fresh"},"measurement_id":"2cbaa731c2cbd72dce7b4099df27d5004681a395331e1cad9984fff7b6a7e3e3","metric_details":{"benchmark_page_updated":"2026-10-01","cost_per_test_usd":5.258418,"date_is_proxy":false,"latency_seconds":1035.191,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-opus-5-5","source_locator":"Results · Overall accuracy","stderr":4.944},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":null,"source_content_sha256":"94ac4dd6ac29ff1fa3171880ba2c27f0dc1234271acd4614665acc1145b359d9","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"bd88c0f6a9924a5a78eb490fba82ee2736f581702d0580b4c3abfffb688a74cc","result":{"confidence_high":100.0,"confidence_low":85.36576,"sample_count":null,"score":95.056,"score_display":"95.1","source_stated_rank":4},"run_fingerprint":"9df443feb624c3b32f9b099251b33f4c86ac489e912205de795b14c487f088e4","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"df16bf720e5579cb295cd091199c64a594fa57a22d5fc3807314572e3c2dead2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"77b1e40d9623ae3a6725809051d0b7b894edc2d9d1b17d903b5dc21c8f471e81","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.8981481481482,"score_display":"66.9","source_stated_rank":1},"run_fingerprint":"b189fe79d6b7cf0b39d2a602d1832ca5b8e498bac87e653b1b87577632500942","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"02684da852cf5e673b6c3ba2b7060d2c0365932df8d4398d618f4522c7ba4cbb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"81a7ca89d3912324159bda7169acd9cec71cd2c2cc0dbe4635924cc0f135f9f5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.0462962962963,"score_display":"65.0","source_stated_rank":2},"run_fingerprint":"cfcc18143d0bacbbe68c946ac43628e4e355bb197102a869afc5d7f4fb6d6da1","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"89e16da54eafa826840759554a74787775e10668078aa8c1d5e8dd37e19465fd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (High, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1fee29dd0d670cb77a4e1dc0bbf5dad25c199d9241c046df473bfefd6e996521","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.4166666666667,"score_display":"60.4","source_stated_rank":7},"run_fingerprint":"52f58c3b78718b4bedf19a7f9158b2fef026d07a9f1b7f23cec0717e34eb8136","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"235b701f5f66c403239ba39797a09db667973cf0256b13b54ad12dc553f0f9ae","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Medium, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"619065255145c637ded08817dce49ce2f3ffc34db82beaa3d9dfaa07bdde34e1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.2592592592593,"score_display":"59.3","source_stated_rank":11},"run_fingerprint":"14285586551ff87f0e22bee443cc9b801bab3ed23b59e554f5af45994443ec2f","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9d1566973e2eb6eb547c55000677379d2003e3451c4e844f35490b9255c44ea","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5 (Low, Default Fallback)","source_effort":null,"source_model_version":"claude-opus-5-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ff576c1bb1d776a95a67e001c05b96dc936b345579fe87d2caeec44dad3a23d9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.5648148148148,"score_display":"58.6","source_stated_rank":18},"run_fingerprint":"23f63f7ca141081317e3e90838d303d8374890647fac4c531c66ea91a272cb7c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1beb76e1edeee5a4d9848a4733c202642b40fc2e36fe8bb672dcd22139020b35","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"697d107ed148c37125ee3cad28aeda87bba2a91fef5f54fbe97dc7ad717ec522","result":{"confidence_high":72.079322,"confidence_low":61.272103,"sample_count":288,"score":66.898148,"score_display":"66.9","source_stated_rank":null},"run_fingerprint":"4f6639c647a991a782645f1cccdfc4c7a9973a4d2bc1da6e1f173020df6c3670","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4d2729a299ce5871bdf8900550741456d81816facd26c0d01a3f5a6a6652ebf4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-xhigh","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"5b1cd0054bfe7adbdb1e7499ec64cb5347b05545a2a0b2b2ccb776d0361c139d","result":{"confidence_high":70.322489,"confidence_low":59.373985,"sample_count":288,"score":65.046296,"score_display":"65.0","source_stated_rank":null},"run_fingerprint":"68cc2b3fbcf843a368de1bf810eb7c7115c7bb0f6e9c60e3c375e458049318e7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a46cf4167d88265b3ad928c2ef6a62dce332d86ddb5ac39f6051ecb10c7c4938","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-high","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"112091bf4fa62bec33605f45be185a855010c03b35a6cb1befad969b0688b944","result":{"confidence_high":65.891924,"confidence_low":54.667174,"sample_count":288,"score":60.416667,"score_display":"60.4","source_stated_rank":null},"run_fingerprint":"4c4bce13821d16ef34aa7c799c58b619b01d2632885cdd1e724afae3ee79abd6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a384740fa459d17f2c5e645e6c951da652020ea8518a02251a4927f5acc7022a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-medium","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1b99362634cca0b21b137522c82d688e14ab9eb4c868757b6385a6ab786a8f16","result":{"confidence_high":64.776045,"confidence_low":53.498709,"sample_count":288,"score":59.259259,"score_display":"59.3","source_stated_rank":null},"run_fingerprint":"76748f618795469a6d55218d828555a9e1cd2486f50624b4041d98e432042625","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"532f5ad52c08125b3acdaa9a40d2b0fd70967cfe6b584f790664311da27e68dc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-opus-5-5-low","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"45e18a5f479ff7dcc88c104b1b5e828b6b50bcd33d247ec966c22e4a6cbf39ae","result":{"confidence_high":64.104981,"confidence_low":52.799166,"sample_count":288,"score":58.564815,"score_display":"58.6","source_stated_rank":null},"run_fingerprint":"885fa491e429b2116263c62d2a3d73f1012a6e696934bab4e7a27634d93487a7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:48Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:48Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ff9657db2e4fa8853b4c943ff7846a347ca56f3f66c93e9b4d18bbc4b492cfbf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.271178","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-5.5-high","source_effort":null,"source_model_version":"claude-opus-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"9482b517aa668f1b1f7e79656e624fc554b91de96483745271e9193220171c39","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2147.38,"score_display":"2147.38","source_stated_rank":5},"run_fingerprint":"a5bf6bf9202d04745975e41353aa5259651b49e919c2c91966eec41a121bb0bd","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"008c8ae40cdb954f9f9c0b3b4506dd1ba848f8e5b77949f73ee10cfdf8ba9695","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":57.922811,"date_is_proxy":true,"latency_seconds":5766.524,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.526},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5-5"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"6e190f681cdb9f0436e26fd39bbd549e68e2a19107325719ed3c9805299b54b5","result":{"confidence_high":93.28496,"confidence_low":87.30304,"sample_count":null,"score":90.294,"score_display":"90.3","source_stated_rank":4},"run_fingerprint":"daa2252c435636c6167eb7870b5ffa32087f26855d211a2a1a6dd6377b89eaf6","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b8a9bbe6cfc610b37bb681b49b1649c0d78b3fc2e3ec2c761569f65f49e451a0","metric_details":{"license":"Proprietary","variance":67.3127766728676},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-5.5-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"65f61f02f3fd6809346fbbec103de045c84acf78753d3ac7651cc7b843435350","result":{"confidence_high":1831.4418914836465,"confidence_low":1799.2810917612887,"sample_count":2062,"score":1815.3614916224676,"score_display":"1815","source_stated_rank":1},"run_fingerprint":"7a760923d806460e778962ecefa53c2188fb2c3d5cae27f8679e3c0768896077","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"32a28fd1936273667c68b46d3d3bf04ee67df7fd1bafd3588eba9377b1dee87b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-5-5_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"0356dd4f595249bce36ea6cf7a986c444e1e42dbd2921d882a37c6a2025fa4a6","result":{"confidence_high":1837.24,"confidence_low":1803.38,"sample_count":1887,"score":1820.31,"score_display":"1820.31","source_stated_rank":1},"run_fingerprint":"48a56f8a57d36cab5681fe4fefcfa5e5da436f2dda577300ef54cf71962a00a3","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"mirrorcode","metric":"mean_score","name":"MirrorCode","release_id":"release_07bf24088c0805f7b6df7a88","split_or_window":"15-programs-30-language-tasks","unit":"percent","version":"ML +Private 2L"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T16:42:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T16:42:00.000Z","first_observed_at":"2026-09-30T18:53:13Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"396a6ebaa54b1ce07711dffdbf6f8b28dc30421854dd22280120a47866b81eb3","metric_details":{"best_score_across_scorers":"0.774","model_release_date":"2026-09-22","stderr":8.309999999999999},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mirrorcode-ml-2l-claude-opus-5-5","log_viewer":"","logs":""},"run_count":3,"scaffold":"Epoch AI Inspect ReAct agent","tools_allowed":"shell, text editor and visible-test evaluator; no internet"},"protocol_fingerprint":"16c548b388367f0b7c76db1763b5889ec1e4d4a986b20227b16a4569ff7d348d","result":{"confidence_high":93.6876,"confidence_low":61.11240000000001,"sample_count":30,"score":77.4,"score_display":"77.4","source_stated_rank":1},"run_fingerprint":"528e65da6b5c67e11099a738905733a7717c3b65ccf0b0c3a529388607d63c22","source":{"content_hash":"3e212e97e03d0343e676ecc7ff046ee7b94d219ba7d863364634b7361dc89b4b","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://epoch.ai/benchmarks/mirrorcode","id":"epoch-mirrorcode","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · MirrorCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/mirrorcode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d6361fcb74537acc4adf2b6eadd36aafe7af5c7a53057a3508c910500b40c187","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.496033,"date_is_proxy":true,"latency_seconds":383.079,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.716},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5-5"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"e0f672c4f201d079170eac8b74b4746eacbaeb27b155a380a331ff285f3589c1","result":{"confidence_high":91.00336,"confidence_low":84.27664,"sample_count":null,"score":87.64,"score_display":"87.6","source_stated_rank":1},"run_fingerprint":"a90263317377696b45c3b606502de2848cea0f80e13953555fd5bf7b43a8afc8","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9db139ba7c2f83c794a97731f5647689086d0ef509d190ad348d0293475cb1c6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.813","mean_standard_error":"4.0","model_release_date":"2026-09-22","notes":null,"standard_error_points":490.00000000000006,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5.5 [claude-opus-5-5]","source_effort":null,"source_model_version":"claude-opus-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"d5cf4b9e7526cbcbf09f2d7615f3df6f116956b0a6969f0cadb856309ae71fb2","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":73.5,"score_display":"73.5","source_stated_rank":2},"run_fingerprint":"1417e3ebeea1e7fe5e0d42752c26623717a1f89b515adfdc5cda365ef6a5a7a4","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:45Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bcc6d73b265964a40a4a4fab9aa2051a689e41df25346ac5d4af502b700f3482","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.657","mean_standard_error":"4.9","model_release_date":"2026-09-22","notes":null,"standard_error_points":560.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5.5 [opus-5.5]","source_effort":null,"source_model_version":"claude-opus-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"c69e5d59b7bc3649b3fd88a205e6ad53ef62a8a31803087ded0d33e292a0dcd2","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":52.5,"score_display":"52.5","source_stated_rank":20},"run_fingerprint":"6ee22ffbbb8ec0d4d6411a7fb8dce6b0069caa30e717cc13316449eeb55a5a16","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"955549fdd158661e23da8bd8ee71469f669baa71c798a00828ca0f73df9d1a25","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":5359},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Claude Opus 5.5 (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a1e1d8729b77a55f21cb682d2a8463cf29c4dd8c4399800bf1376c9b6c847f39","result":{"confidence_high":0.15988541029765996,"confidence_low":0.11645483256257871,"sample_count":754031,"score":0.13817012143011934,"score_display":"0.1382","source_stated_rank":2},"run_fingerprint":"91b262911d54e878559f9dd0c1e6c9d372a24fd1f139c0f07f3fb3ae622aa8de","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:48Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:48Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f268b030efd7052cfb06a149ae65e3086f5b270cf78651c50008d1c1ba76a216","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5.5","source_effort":null,"source_model_version":"claude-opus-5-5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"b9e308c17db60b50ad2b30391faed42194797f8c3a82dca4f1f5b099cdb75f72","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":9235.248333333331,"score_display":"9235.25","source_stated_rank":6},"run_fingerprint":"9b1126ae411281d711d3eddbfc691588e90b28d7a5b97ad8e2762a4625db8c7e","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3a3f12ef819b20354dad02a8","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b4dcebf0ec6ed6cb76fa2b0d341052e71c743b1c7c9f456cf1380f504a9006b5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5-5","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"715cb502c8e02a3af7ccc1bdb72e156f397cceb402a1a4cb0483ab552abadd43","result":{"confidence_high":1869.4,"confidence_low":1822.93,"sample_count":220,"score":1846.17,"score_display":"1846","source_stated_rank":null},"run_fingerprint":"ef8df215979c7e499969695017db9586d95de951fd5b4df98835392d14107e8e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_644045ef3ccf710aa2a97573","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7b604c5bf447f885df78229533ec6a7aa51a354c63fbaa45601a0c32768b87de","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5-5-xhigh","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"405a1e5cf95caa81a4262243dd51ae4f733b48bb61aa07a99ab964b5430185aa","result":{"confidence_high":1842.19,"confidence_low":1798.02,"sample_count":220,"score":1820.11,"score_display":"1820","source_stated_rank":null},"run_fingerprint":"f8a25569b11bb4485ea31abd4772fa4cd752e66cf9c37e38c076439df90404e3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_ac8995612553984b32210e23","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dae90ee0748d2597bb998ce22f10e62acc13431a58073338e27cbd42e2cf0f70","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-20 / +20","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5-5-high","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c009e31513f129bc0f0c7d8f8d6cfb34fa57bb391a0d2708405e9ef71df086cb","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1691.72,"score_display":"1692","source_stated_rank":null},"run_fingerprint":"fd94088319053b4d60b671f6b54adc0d4f88329c6e8b2ff31f557e668e4f82eb","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_37e371ca4768fbe805410768","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4a65b08cc250e06da7a4b414e2d1568661fb1749746d5d45cc5b832368d92355","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-19 / +19","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5-5-medium","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"1bf3c9607fe21e9c0893806595f4dd8fcd5b5db95379fdd768b779e5360e2d4a","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1576.2,"score_display":"1576","source_stated_rank":null},"run_fingerprint":"c5ab7224835f8dab76f8a7f49a50a811ca7d84f793fc70c724fafe94cea7c72f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9bcad8178005dc3a63c077d5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"96f44d5485870eebf95e28e4fb06479124eb0b9df13820224bcc0bc10203240f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-18 / +18","source_locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-opus-5-5","name":"Claude Opus 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5-5-low","upstream_model_label":"Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"6c6d193bc5e46deb7e41ccbb2e38187cb54db413da1e620e4ea1cf64f1f2bd22","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1223.51,"score_display":"1224","source_stated_rank":null},"run_fingerprint":"78b926f855e452eecfa4d88eac8716eec4f78efc3853b196db625310cf41a6ff","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c59f0eec189947d313eaf227","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9f79ac599e96507ae8a31f7cfb58177a1833c148c38ccb23c65a4dc9c5a8f3c7","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-09-03"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-6 Astra; model release: 2026-09-03","source_accessibility":"API access","source_model_name":"GPT-6 Astra","source_model_release_date":"2026-09-03","source_model_versions":["gpt-6-astra_high","gpt-6-astra_max","gpt-6-astra_none","gpt-6-astra_promax","gpt-6-astra_unknown","gpt-6-astra_xhigh"],"source_reasoning_efforts":["off","high","xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b3614cc31d52c3776d4f1ed063a1a27b0e7537b350935603160bb85f9bc673fe","result":{"confidence_high":171.05,"confidence_low":163.14,"sample_count":null,"score":166.51,"score_display":"166.51","source_stated_rank":2},"run_fingerprint":"88bdac77aa745f08c8c8d3f0e4aa28ef61e8049631cbb813ec26661e36e47828","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9276dc96b9e1782b83ae6c86","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"05cba252f7c8d7a24e7c2de6bc863d5f4e1d10ec6e9652c8389b4550ee9e4586","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Max) · intelligenceIndex"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra","upstream_model_label":"GPT-6 Astra (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"8d018f472149a43a27777a069754d56da02a749fa999ac8e43de10bfa72b5d27","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.673669,"score_display":"52.7","source_stated_rank":null},"run_fingerprint":"bcf8f870fc9c11dcbb977a2c850ee0d555f160d8baa8f9a076cb4e7932fbfed9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"481cab42f6c9512a87bc576d3881460e65056574caf3adba0e0f805b9dfeec67","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · intelligenceIndex"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-xhigh","upstream_model_label":"GPT-6 Astra (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"18f23fe7ddd3772a848bf38f7b19a21771f6f5bd946e5c848cb3c351443cef22","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.386328,"score_display":"52.4","source_stated_rank":null},"run_fingerprint":"e14d9b932b96019edd037a571c010d8e39151c6a481070c7b206d83244e55457","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f9b17c35632955cc87e7854f616a9ca14f666d3d2209167f03d79c83f4a74714","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (High) · intelligenceIndex"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-high","upstream_model_label":"GPT-6 Astra (High)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"1ee9cbfcb5e6b9996025e61409cac7b3ee6d2d08ed60d4f7a38e299ff56a452e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.919147,"score_display":"50.9","source_stated_rank":null},"run_fingerprint":"3ce997509049de3d40e968c3d80df066817f456eac7ee05170a2e84d505ad2aa","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (High) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9e7f48fd0b8890e9c33308056f02801ad2b59653ce2d88fb68bbc78ab92e24b6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Medium) · intelligenceIndex"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-medium","upstream_model_label":"GPT-6 Astra (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"5f8c85cce4e16305923ecfa91017697fe742e6f9108ac702af2f0840b03d27fb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.570436,"score_display":"49.6","source_stated_rank":null},"run_fingerprint":"496487b7dd38777a74c0f79784253e1b80b0b8416c324751e42317221d5e2122","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"41017d48f457ba2889474e0fa2432474387d33784476893311f19dc7ed961ba0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Low) · intelligenceIndex"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-low","upstream_model_label":"GPT-6 Astra (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"2148bb60f2c87e524df7834c9551ae8f4e6f7caee4b8e3172c972a11a21a43b2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.781924,"score_display":"45.8","source_stated_rank":null},"run_fingerprint":"50a68825ecccc2e3a4c93c806bccdecbfe02e0afd3e7f087ee1af369a73b2c1d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"17aeecb8f5e07d3d0bcc14e0415db615910548b1e9effcf8631790d3708c6707","metric_details":{"license":"Proprietary","variance":12.256811302579537},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1483.9726885799187,"confidence_low":1470.2491269588413,"sample_count":9156,"score":1477.11090776938,"score_display":"1477","source_stated_rank":29},"run_fingerprint":"e2e36bef1589cc855d524eb9d36e806700c4f01f812e21e0eceda81e8ac80b74","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:11Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"acd4979dcfc6f69c0c0d707a03dad947ad87d0250c77636115ef2bec571bb52e","metric_details":{"category_scores":{"Agentic Coding":57.323,"Coding":80.35849999999999,"Data Analysis":82.97333333333334,"IF":75.57900000000001,"Language":89.43066666666668,"Mathematics":96.80624999999999,"Reasoning":92.65375}},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.16064285714288,"score_display":"82.16","source_stated_rank":4},"run_fingerprint":"561c4d3c6dcf61deabde1e56a58720510d465e061af01933631a7a5b773e9dae","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":25,"at":"2026-09-09T18:59:21Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-09T18:59:21Z","status":"fresh"},"measurement_id":"e18f1947dc0dbf8360546cb544e5c3d1d200083c430450d86e43a1e9b59aacf7","metric_details":{"calibration_error":39.0,"confidence_interval_half_width":1.94,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=GPT 6 Astra; createdAt=2026-09-09T18:59:21Z","source_row_created_at":"2026-09-09T18:59:21Z"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"GPT 6 Astra","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4d088526835dd9370a0d7179f110d008448a3b5fa887e75f91c52a66a221a3a7","result":{"confidence_high":56.739999999999995,"confidence_low":52.86,"sample_count":2500,"score":54.8,"score_display":"54.80","source_stated_rank":1},"run_fingerprint":"3faa03123beeae48e78f8b7a0bcc878b813c6fb496067ae8bb4872045055056c","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=GPT 6 Astra; createdAt=2026-09-09T18:59:21Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_38556cdaf0f3888ee868b8cb","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"78957e061817f147dc09e38b7c1206313e9000efc1479803eb4f23c897a4363c","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Max) · hle"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra","upstream_model_label":"GPT-6 Astra (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6646ff1d63a3c5eb50aa133e02bd4d08ca1eedda7a43a31bbd8a86ea4ca4d30b","result":{"confidence_high":56.77043,"confidence_low":52.573454,"sample_count":2158,"score":54.680259,"score_display":"54.7","source_stated_rank":null},"run_fingerprint":"ae9752f96b10ee99dd9972b99f78317bbfb4c2f78eb17936909045e0663748d6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0fb1131e67fd6c18b04d4d8fcdfd9c2bae96be704c7c1ee057ed70135a7f868b","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · hle"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-xhigh","upstream_model_label":"GPT-6 Astra (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6a08f3e1d4a42e1d3bc96ac36d368ec9cac650eef26086ab879db6ee0b7bd568","result":{"confidence_high":56.67828,"confidence_low":52.480578,"sample_count":2158,"score":54.587581,"score_display":"54.6","source_stated_rank":null},"run_fingerprint":"945c0a2856e32f9da2d988e050e28996cdd0f9f289e78b056ddd063c093b2ae4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"48fc62a3391512ce17f36298fa76b76f3494488516cb685770be41cb33eae866","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (High) · hle"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-high","upstream_model_label":"GPT-6 Astra (High)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f238930e8a1d126337ebd667bca3191f19d96e4f22e08018f2dd9eb2cba29e5a","result":{"confidence_high":55.156738,"confidence_low":50.949166,"sample_count":2158,"score":53.058387,"score_display":"53.1","source_stated_rank":null},"run_fingerprint":"da949c4afa2d99853a1c80b2aa75f473141c892f7de90b855a382bed3a0b40ea","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (High) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"044bc442577d3a0b183bcd9c4fb8630873963627907fd0dc9b91b3cb78da7ede","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Medium) · hle"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-medium","upstream_model_label":"GPT-6 Astra (Medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b808564f96bea5ea728961b27b5bb146241fe4f3b7dcec6599ce2c66ddbab3ab","result":{"confidence_high":54.833733,"confidence_low":50.624576,"sample_count":2158,"score":52.734013,"score_display":"52.7","source_stated_rank":null},"run_fingerprint":"2266191138e2fb627279c228dd0d6eb5992c5d60bafb4b7bbb93be3c7c26473a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"abbd7938fa048f8f95b3cbb5148bd87531953298e79556036aa457f740260c3d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Low) · hle"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-low","upstream_model_label":"GPT-6 Astra (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"80528b9b71a2ed304987a71ddbec2fb1c678cb0bb30c603ec5806ac73941b5e5","result":{"confidence_high":51.321099,"confidence_low":47.106169,"sample_count":2158,"score":49.212234,"score_display":"49.2","source_stated_rank":null},"run_fingerprint":"1baf8c9abc54031e2bb246ab62337f3245e1bbfe5dbb36e81e3f2f1fd41ac6ed","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T14:57:43.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T14:57:43.000Z","first_observed_at":"2026-09-03T22:50:14Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f7f0cbe5530492cd1674906ac54291393f14f494b902f3c983e338fabe8b1ad7","metric_details":{"best_score_across_scorers":"0.756","model_release_date":"2026-09-03","stderr":1.3588548437881387},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"PppwnX7EjXx2SeTcFN8Ujn","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"2854e3d43358f88d6e47f693d81162b933415b351afa2cdfc215f08a78adcd61","result":{"confidence_high":78.26335549382475,"confidence_low":72.93664450617524,"sample_count":1000,"score":75.6,"score_display":"75.6","source_stated_rank":1},"run_fingerprint":"70aa1dbd3de3e074dcf84a623350a47f076b1724adc43a97421b5b34d674103e","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":18,"at":"2026-09-16T16:53:42.381622Z","basis":"evaluation_at","evaluated_at":"2026-09-16T16:53:42.381622Z","first_observed_at":"2026-09-17T14:33:47Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aa35c35c243cdfb35ac63243aa18c4dca9ce9bb909e5b7a18f624a4ccc34b87c","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-6-astra","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ec92e842ffa936440aff832f21d3d3b3440073b21926c3ae94d8fdf9bbd3f1c8","result":{"confidence_high":78.37730044968421,"confidence_low":73.23395274724672,"sample_count":null,"score":75.80562659846547,"score_display":"75.8","source_stated_rank":2},"run_fingerprint":"424fba1a8a3c641d469da0f4e2a4480ed2808e06e8dce8c5def565b384674e51","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_502d9746cf5b263d42f5c6b0","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"982c4703458f9eb40206cfd530fedb54ae662649f11399ed5bf945d295552efd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Max) · gpqa"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra","upstream_model_label":"GPT-6 Astra (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"33949a9f271b2d1613d3d9900dfb3c0d34fa720aee980600ef9843b1db2a6183","result":{"confidence_high":98.007227,"confidence_low":92.360665,"sample_count":198,"score":96.060606,"score_display":"96.1","source_stated_rank":null},"run_fingerprint":"72300eab7933461bbf3503c1ad08b32473feee602dda29560092b80123958b91","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Max) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4eca12fb0b2ce96ee9f0fc58548a40c5489015e03b338e8be8c4ee15dfbd83a3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · gpqa"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-xhigh","upstream_model_label":"GPT-6 Astra (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1f10d95eb7c9ac5405c51929015de8eb7b3681c260979dd2fd4cb386187d8dd2","result":{"confidence_high":98.143043,"confidence_low":92.621199,"sample_count":198,"score":96.262626,"score_display":"96.3","source_stated_rank":null},"run_fingerprint":"5246fd769bbc4754192c8fd3794f6f46eef5a917532fdc44290a7dca789ad583","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c05025ad63b93ed5c59802c6d6be55bda65c70c77eb1f717c41b4b9f9d4c9c7e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (High) · gpqa"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-high","upstream_model_label":"GPT-6 Astra (High)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ee6d8ef9229e02555252a0b6850e5efba380ee963157ad53086e934df1727d3d","result":{"confidence_high":97.233874,"confidence_low":90.954091,"sample_count":198,"score":94.949495,"score_display":"94.9","source_stated_rank":null},"run_fingerprint":"59474688fe9253f512a2688f94678ab116e0ca22da6062a0ee07ebc9fc25df4a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (High) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"59b39049c141a1eac045ecc565ae080445a3af4bfccf597dd8530ab3475c6a90","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Medium) · gpqa"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-medium","upstream_model_label":"GPT-6 Astra (Medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fb3db032495a6a98356bdf27621048616ec225e3152c29b38024c70b24a51978","result":{"confidence_high":96.49947,"confidence_low":89.706743,"sample_count":198,"score":93.939394,"score_display":"93.9","source_stated_rank":null},"run_fingerprint":"3cad3770ba860788fffbb25810b2b7274961d7bf2e2b94b5f6e4a0474e5c54f6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6045b842c40635a84fba3c57a3d3b057f9335a22ef1e6a405ca12cafae3c7497","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Low) · gpqa"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-low","upstream_model_label":"GPT-6 Astra (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f2bab04c52e8058270cbd51b63c506af170502cb53a0e38496dd275d0b559909","result":{"confidence_high":95.894948,"confidence_low":88.725863,"sample_count":198,"score":93.131313,"score_display":"93.1","source_stated_rank":null},"run_fingerprint":"78490148945875b0593f2eeba35cf6cfb8b28051d5cc1f7eaed7d2dccf615079","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":35,"at":"2026-08-30T14:57:43.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T14:57:43.000Z","first_observed_at":"2026-09-03T22:50:07Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"01c3c08b90214ca8d74fe0d90115b9f2c1fc3c84011d60d0bf446b2b2d98e9e5","metric_details":{"best_score_across_scorers":"0.9577020202020202","model_release_date":"2026-09-03","stderr":1.3696812097020312},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"3h8STJpG2VzY9LiBUzwotv","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"1e51b954d9123b3d2ab2d28f2f74adea5c3d58348411ea1db5d8ccadecef718b","result":{"confidence_high":98.454777191218,"confidence_low":93.08562684918604,"sample_count":null,"score":95.77020202020202,"score_display":"95.8","source_stated_rank":1},"run_fingerprint":"e93b713e9519ef146668fb31891d673317d41de696cc9a570ad16357e36368c3","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":19,"at":"2026-09-15T23:53:31.157098Z","basis":"evaluation_at","evaluated_at":"2026-09-15T23:53:31.157098Z","first_observed_at":"2026-09-17T14:33:47Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6bbfe99219e73a335645342c684a6cd5b772719bd008a8e82d459d09aea0a045","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-6-astra","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8d1858a6657abe66d2803657af368107942c6c6273c383544ca046fb21694c1c","result":{"confidence_high":97.26290708398315,"confidence_low":90.61588079480472,"sample_count":null,"score":93.93939393939394,"score_display":"93.9","source_stated_rank":1},"run_fingerprint":"5377bfa5db850ed0560764b95621bbc1c6584cbadeb080a29c8cde945b765e1b","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c3d59d8e80dbf3e0f70ebf92","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T14:57:43.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T14:57:43.000Z","first_observed_at":"2026-09-03T22:47:19Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"98598aff871ec98417280a97c0e0eec68508a2348de4900b1c8e86f938978ea0","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-09-03","stderr":0.0},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"L7kW8yxFKCjVXhxNnt3VTq","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"966adef4976badfac20721ce0a1dd126f88966c8309933d9a2fd20e6aff1f86b","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"27406871e761b09e750de994e3cb6467080e0dbc35c192b185523af638a8b5df","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T14:57:43.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T14:57:43.000Z","first_observed_at":"2026-09-03T22:47:28Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7e74473e5ea197dce36292e0d0fb9f5683a9b7b331b711d10f24ea45248dd9d9","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.9368421052631579","leaderboard_private_problem_count":285,"model_release_date":"2026-09-03","public_example_count":10,"stderr":1.4434038295343787},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"csK2fe5T8p2R7WgoJUu6bZ","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Astra (max)","thinking":null,"upstream_model_id":"gpt-6-astra_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"313848536ae7469ee53b15def714d6dcc5afc20f13f3509167c3e5b0871cedf3","result":{"confidence_high":96.51328203220318,"confidence_low":90.85513902042841,"sample_count":285,"score":93.6842105263158,"score_display":"93.7","source_stated_rank":1},"run_fingerprint":"9c36901d7412a0eee9408a837b1a61a152b8773ac1a0919813ee455998d96c40","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-03T22:49:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:49:18Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4a13d3c242a8f8eb14174ced56ce4304e6bea70da20624b01662b1def3f9a740","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.1194680833333335,"model_release_date":"2026-09-02T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-astra"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-astra","model_type":"CoT","upstream_model_id":"openai-gpt-6-astra-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b28d063f4e06ef5ecce16f5af8d1f6ae972c1cfcc5d40ff95a612ee586729b39","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.0,"score_display":"95","source_stated_rank":2},"run_fingerprint":"dd55dcf84052fe4ce1d729f075d115b463cc2722918e48512282b3fe45e944ed","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-03T22:49:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:49:18Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"65b65bdca4024b21a02bdf3247a25fe85ff78679339bc72c4603258a52b4fa12","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.8286855833333331,"model_release_date":"2026-09-02T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-astra"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-astra","model_type":"CoT","upstream_model_id":"openai-gpt-6-astra-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6a227920da36b4bef952b2c55263f85e79486b8ddd3cae024f2a5d4b1b382ffa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.33333333333333,"score_display":"93","source_stated_rank":4},"run_fingerprint":"9e268b1c2a245521b82794f76ef2be641bfa636b94d3a5dc4a10441c8e304e44","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-03T22:49:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:49:18Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f0883aad311c03208f9415bdd36ca4a44fbcaca756de6f5d8a0c6cc0f150b355","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.6680793333333332,"model_release_date":"2026-09-02T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-astra"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-astra","model_type":"CoT","upstream_model_id":"openai-gpt-6-astra-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cda509e6d70f73a6165ecb4f02c39b8c79e6e6a5e2178ef6ad08d1af7abfd753","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.08333333333333,"score_display":"92","source_stated_rank":8},"run_fingerprint":"23b74a1f027ad6d10ba466b0606b416afd92d3e0ae58a0c50b28f19beedd9cbe","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-03T22:49:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:49:18Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a16773f78ce7f2dcffded4a430607ed2727f3707e934b2abcf6c4fa8c0b04f20","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.4800926666666667,"model_release_date":"2026-09-02T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-astra"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-astra","model_type":"CoT","upstream_model_id":"openai-gpt-6-astra-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"950ad14558720e94ac43ff552963e69d42d6836d7fe0988b20b0db243628d93e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.08333333333333,"score_display":"92","source_stated_rank":8},"run_fingerprint":"6897cba2f394bdc050b5d0a6f39b9c22f27599c89e2baa8274a59b9d22c6a262","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-03T22:49:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:49:18Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2588eb8afa8716da0fab65d869fa7a9829f9d1f4b5f1016ce71fd01fa747784a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.4162418333333334,"model_release_date":"2026-09-02T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-astra"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-astra","model_type":"CoT","upstream_model_id":"openai-gpt-6-astra-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ce597c476fa16968337098c0ec58dbc6e729bcb7994e3744c726aff093a5192c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.41666666666666,"score_display":"85","source_stated_rank":28},"run_fingerprint":"f2ee0daaad94bc1c4219d6b17162a4c784d56f335b238d70b409864d10d0a90c","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-03T22:49:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:49:18Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c0dfbdd6afcc9346082210722acde5e0336a45e382c80bd808fcd42dd2226864","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.3696743333333333,"model_release_date":"2026-09-02T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-astra"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-astra","model_type":"CoT","upstream_model_id":"openai-gpt-6-astra-none"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fd2a06a7f9d2a3ab0a4963a10dc70a6e187486951aa54896d8478bfee725936f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.583333333333336,"score_display":"60","source_stated_rank":79},"run_fingerprint":"13c2f5702048eec4bbc320eee1e72992e23a21cb31a4cf9efd06d79ecfd81a26","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3807ea10a6fe1e9bbe8e157c","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"417103f7066b795c96dcf609bfa2535f0e9abee740dc770bf456b36bda3d80bd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (Max)","source_effort":null,"source_model_version":"gpt-6-astra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"95accef2383954f540c04f28e9fac3dc19ccd3a57e2bf0e61650b3da110d6c2e","result":{"confidence_high":96.814195,"confidence_low":92.235545,"sample_count":360,"score":95.0,"score_display":"95.0","source_stated_rank":1},"run_fingerprint":"8ed2f63dc097ae30197c79944033595d54c36fe8bc5dc10af74028fe7078ac48","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d4090bb6b17405b01686d5a480127704c3918af33c97a88d69fa93ff67a11c87","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (XHigh)","source_effort":null,"source_model_version":"gpt-6-astra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"764fa7b58138e0c0fd2d58331695ebd48fe672eb76c22e1dd92293b516edd949","result":{"confidence_high":95.479458,"confidence_low":90.272143,"sample_count":360,"score":93.33333333333333,"score_display":"93.3","source_stated_rank":3},"run_fingerprint":"dfe1e4df376fc048cde73f2be854329d92016f45afb1be5a0835df98b11998c4","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4989e65e240a90f5184d9f8763a1614fcea63dc1aa1a3b13ecebbcdac205979a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (High)","source_effort":null,"source_model_version":"gpt-6-astra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"663ea1eff70fe61eea48f5275b4632be161d540710a2d4ddf2e7ea7941d29ea9","result":{"confidence_high":94.448708,"confidence_low":88.82929,"sample_count":360,"score":92.08333333333333,"score_display":"92.1","source_stated_rank":7},"run_fingerprint":"0db23ac6c3de85714115f458fc169c0fe5574866859153d397c7d5b4e8a87117","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2d0b967ba9dd778c6fefb347ef2c3a1ec0054ca0a89095fd453e960e5a266bd0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (Medium)","source_effort":null,"source_model_version":"gpt-6-astra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"1d6f948e9188cf8b8860ea8157b9b71c9dbd2ce30448f07fa47a3ec38be3b12f","result":{"confidence_high":94.448708,"confidence_low":88.82929,"sample_count":360,"score":92.08333333333333,"score_display":"92.1","source_stated_rank":7},"run_fingerprint":"d860a9b2395302bbdfdd7c455b17bf6cc73018657cea6efe68f2f1d888e63fff","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7d0657925a1abf5334369c45bf9f19d3c1c63881da6ca9f232814369f095d766","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (Low)","source_effort":null,"source_model_version":"gpt-6-astra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"2a3c4a8e0fc174132f1df0462a3512a6ee47af0a613d8b2808ce163c169c540a","result":{"confidence_high":88.68855,"confidence_low":81.396894,"sample_count":360,"score":85.41666666666666,"score_display":"85.4","source_stated_rank":28},"run_fingerprint":"5b32a04a961692fd8303ec8e5ae2a5910119ea7d180badb6643f360d0affd679","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"26b37d11cf6e8514db5b3e435392362b86017615493c3fee06e0b6d492cdc73a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (None)","source_effort":null,"source_model_version":"gpt-6-astra_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f446e7d6c0d046bb9f84a6694d01e7ef8195715fdf42d035923918265fc855f4","result":{"confidence_high":64.525624,"confidence_low":54.438672,"sample_count":360,"score":59.583333333333336,"score_display":"59.6","source_stated_rank":82},"run_fingerprint":"6d68ace9b3e25aae02f9daf5a71d95226a7567184dea62eef83252e42668e9cf","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3807ea10a6fe1e9bbe8e157c","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-30T02:22:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T02:22:12.000Z","first_observed_at":"2026-09-07T16:34:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e19c686c6362058595251bfabb71a29627fdae0c85eff3c3f8bd6514779f702d","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.976","leaderboard_private_problem_count":41,"model_release_date":"2026-09-03","public_example_count":2,"stderr":2.4},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"f7BCT27LzcAtaLDP2tcrWF","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Astra (max)","thinking":null,"upstream_model_id":"gpt-6-astra_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"04a3a0352ec5ad74f3c30888cdf8e8bb18489aa80e36e7b0c0f77af63d2842d8","result":{"confidence_high":100.0,"confidence_low":92.896,"sample_count":41,"score":97.6,"score_display":"97.6","source_stated_rank":1},"run_fingerprint":"7760dfa34aabfc6f9ba7a349eacc26b56dc70b7600bbb630f72e74cf26159bfa","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-30T02:22:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T02:22:12.000Z","first_observed_at":"2026-09-07T16:34:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"19c5a728b4f2cb13323da8e7cd1688ef17c50d6bf30353709774ef52f99bbd59","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.976","leaderboard_private_problem_count":41,"model_release_date":"2026-09-03","public_example_count":2,"stderr":2.4},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"AfKJ2DGMhkjZmeFZxGoY3w","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Astra (xhigh)","thinking":null,"upstream_model_id":"gpt-6-astra_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"70dfb499196a292a61285460a28a7922ce85a084f9082271d4eaa18046ab22bd","result":{"confidence_high":100.0,"confidence_low":92.896,"sample_count":41,"score":97.6,"score_display":"97.6","source_stated_rank":1},"run_fingerprint":"88c4f5e84f825415845f0cf8e34fde3353286239415f04b7c0565f41da6d2053","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-30T02:22:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T02:22:12.000Z","first_observed_at":"2026-09-07T16:34:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e619e985274fad127ccca66aa6102f1805bbf94208e650f70840cc49dfab6fe1","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.976","leaderboard_private_problem_count":41,"model_release_date":"2026-09-03","public_example_count":2,"stderr":2.4},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"f46NQY9jSxyKqo9qzrmk6F","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Astra (high)","thinking":null,"upstream_model_id":"gpt-6-astra_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"2543689e4212fb61eb6b659db41b56880d75c62b0032a30c8d1b4be1d9fef4b8","result":{"confidence_high":100.0,"confidence_low":92.896,"sample_count":41,"score":97.6,"score_display":"97.6","source_stated_rank":1},"run_fingerprint":"05ecf4dab15f3da16456c51f5997e3336442bc30dcfaa3a6ccf0bf35c1ccbcc1","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-30T02:22:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T02:22:12.000Z","first_observed_at":"2026-09-03T22:47:32Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f6cf0fad9fdc856fa51a49e3a3a921739af823e873f902964f2cb84ed7d1cbcb","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.975609756097561","leaderboard_private_problem_count":41,"model_release_date":"2026-09-03","public_example_count":2,"stderr":2.4390243902439024},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["medium"],"epoch_id":"5sNRWemuVwb4adBaSwrLFf","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Astra (medium)","thinking":null,"upstream_model_id":"gpt-6-astra_medium","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"1dd3570e793d316c583566001953fdc3dee6780875ebe3b77536a56202748e50","result":{"confidence_high":100.0,"confidence_low":92.78048780487805,"sample_count":41,"score":97.5609756097561,"score_display":"97.6","source_stated_rank":4},"run_fingerprint":"31ab7886224f8f9bcb753379b355661ebd7274f5114962726bc6cc5f0f3f7d58","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-30T02:22:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T02:22:12.000Z","first_observed_at":"2026-09-03T22:47:32Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b7b5b3009a9b6730bd4aab4d3435d8bdb0948177f212c4bccd1c971105e1f3c8","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.8780487804878049","leaderboard_private_problem_count":41,"model_release_date":"2026-09-03","public_example_count":2,"stderr":5.173952057462543},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["low"],"epoch_id":"kRPSnHnDdheajNdqmKsMjs","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Astra (low)","thinking":null,"upstream_model_id":"gpt-6-astra_low","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"00a95277755ade21959da6afcb243fe8442eba61cc38f1ce920b129c01a9e153","result":{"confidence_high":97.94582408140708,"confidence_low":77.66393201615391,"sample_count":41,"score":87.8048780487805,"score_display":"87.8","source_stated_rank":8},"run_fingerprint":"3f830809ccc17f8454e87598f37c2c07ca3bfd48f016700b034a7f4d59dc1f7f","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-30T02:22:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T02:22:12.000Z","first_observed_at":"2026-09-03T22:47:32Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ea3afd341cab93edd96d7af0c6caa900de9a6eb6b7b17b220ddd6b4b76c84e2c","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.8292682926829268","leaderboard_private_problem_count":41,"model_release_date":"2026-09-03","public_example_count":2,"stderr":5.949419959829497},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["none"],"epoch_id":"4BCZTu7HQKmhdwRR2W4bzT","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Astra (none)","thinking":null,"upstream_model_id":"gpt-6-astra_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"491ae820d70af20336f372664db748c7e0c9188422cf36eb72f9b83010b0646a","result":{"confidence_high":94.5876923895585,"confidence_low":71.26596614702686,"sample_count":41,"score":82.92682926829268,"score_display":"82.9","source_stated_rank":10},"run_fingerprint":"56969e15c0b7d9b69bc6168ed3ff5760f6749771917a72c8907c8bc55883f02b","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3807ea10a6fe1e9bbe8e157c","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"76d487c941e64e669aed89df2e682fe98f4a3e57d3f795827e5e0b2fed670006","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Max) · mmmuPro"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra","upstream_model_label":"GPT-6 Astra (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"89585c82fde49b173a5ff7ffefb3a8103767ac9fda2c5a36da49ccf4290abd73","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.878613,"score_display":"86.9","source_stated_rank":null},"run_fingerprint":"4aa1beaed5def4e16c15fa731875e9621e9378429958b7cbf36b09c8dec9b912","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Max) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"120b7f8b607859031ca42b7219aec84581cf4064f188f62cbc26c57061c3df6a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · mmmuPro"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-xhigh","upstream_model_label":"GPT-6 Astra (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"47f51d482b0c04abcddefb4443224990e7c477a7db11bcc5e3306d24bbc18962","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.242775,"score_display":"86.2","source_stated_rank":null},"run_fingerprint":"69c1e078f659354ac96596ffd907c19331009ac3adf2231dfb724acb97bda69c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6764f905bcaf57e472328228e6a271d3adc2a587de4069607b7caed6b2c26afe","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (High) · mmmuPro"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-high","upstream_model_label":"GPT-6 Astra (High)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"67c03eab9275a3bb3805d560785d2096c348421d0ba8bf4dc2c91899723a87a7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.416185,"score_display":"86.4","source_stated_rank":null},"run_fingerprint":"b9e83282f5486eef77dd07de77ff3872c4a60c2a4b571e673c8c8c6d33e5cf09","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (High) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"35d842984dd84e78afc42f19d4c7001d9f84e84ca5b1e1bafdfb45efa477aafd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Medium) · mmmuPro"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-medium","upstream_model_label":"GPT-6 Astra (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4c8e31fa146b2738926068be690e62231a210a8549ed29d7c16ee8aeaa80d188","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.086705,"score_display":"85.1","source_stated_rank":null},"run_fingerprint":"b410fb68ecb18089d8ae06a687a2c9906faf43d12fd02a7fbd74c28e1305c5c9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"87464065f09a6248fbcf665e7fb5e7cf465372d4205d737a8966e467e5ef85a4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Low) · mmmuPro"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-low","upstream_model_label":"GPT-6 Astra (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1e39e2624213e7ddb00e89ae9ddda29c8213d6c187bd71a61aaf0cd853c6c611","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.624277,"score_display":"84.6","source_stated_rank":null},"run_fingerprint":"33cbbc57e9773dc76613b2dc4312affb0f645b88a40a746e04749425d6e6e57b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"eeb5ecea119f2f11e81a789b9ff2c5edc1b9ee8987dced2afdf328da15f04e56","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (max)","source_effort":null,"source_model_version":"gpt-6-astra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b2b82929dca7ccc0951efc9395ca4a5ab3dc1a12707f3184ef0887dd09b8333e","result":{"confidence_high":43.237892,"confidence_low":22.067881,"sample_count":71,"score":31.7142857142857,"score_display":"31.7","source_stated_rank":2},"run_fingerprint":"ca7b244da2c862c22f0639710e30c2a0c79012badd86bc099f315b1d1c61f113","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ca74298f2060ca4b7898580580092bb4dab310d88b13f3956b58792d49716d8d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (xhigh)","source_effort":null,"source_model_version":"gpt-6-astra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b93ccd2ec7039d45813df7ccc263862931ccef3abfdad0c315c51e4bc06a719c","result":{"confidence_high":42.942593,"confidence_low":21.821083,"sample_count":71,"score":31.4285714285714,"score_display":"31.4","source_stated_rank":7},"run_fingerprint":"04eda15f118d8b475b8e9dbb2c631e2d0bbf519905ec83cb1c770367ade0bea8","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"51b30a65b203213b9fc0dd8cdf36a73058a4f79126217eafadca53e5b9585fa0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (high)","source_effort":null,"source_model_version":"gpt-6-astra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7598415d83efcec188ed25ab475430a3694e9c5fa4152653497ab2f4cd460337","result":{"confidence_high":40.265031,"confidence_low":19.61977,"sample_count":71,"score":28.857142857142897,"score_display":"28.9","source_stated_rank":22},"run_fingerprint":"794ef2a65bcaad4dc040c881aedcf9d52701c6fd4eeefa3352f7859753cfe1a2","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"814b859b3ec1ee6693e70060703f018a6d766572a945a8be382818b1938cbd5f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (medium)","source_effort":null,"source_model_version":"gpt-6-astra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"11492df8d7ab325d81d474e99f9651b90b37261ea19f1c3f0dbd8e6b7e62184d","result":{"confidence_high":40.564344,"confidence_low":19.862553,"sample_count":71,"score":29.1428571428571,"score_display":"29.1","source_stated_rank":19},"run_fingerprint":"4acec52a11c91d4959d05de9299cf0a6885cea2e1d500fbb120f065b1dd3572b","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f645da5b1d9332787a7a6e641ce31d9ed7b3c43f1a4c805559310e0fe5a2a088","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (low)","source_effort":null,"source_model_version":"gpt-6-astra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"24ff57671fc9fcdef41950e003a75a337e85007d10367f262f270c15dc2ca943","result":{"confidence_high":37.549846,"confidence_low":17.456079,"sample_count":71,"score":26.285714285714302,"score_display":"26.3","source_stated_rank":34},"run_fingerprint":"98302fc17ed5ca250138cb46b7a9fa47c3dcd0cbd4aee5110787e26c40d3a8c4","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"166bb8efa7f7e65775c5152dd5c3877fafc4fbb1fcc824ad5ea43768dfa724a1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (Non-reasoning)","source_effort":null,"source_model_version":"gpt-6-astra_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"33c005107ca57e0d09d255465ddaf87b35aebb4d65ccef016e69d3f043824fcc","result":{"confidence_high":30.415434,"confidence_low":12.122254,"sample_count":71,"score":19.7142857142857,"score_display":"19.7","source_stated_rank":54},"run_fingerprint":"73d01fb4c0db7c3dbf44646ca8efc965fc48297da2add108a1004c86cce3df14","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3807ea10a6fe1e9bbe8e157c","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bc0856375ae297056dbc7846060f777a1fd43e9f101aac48a7098430d4a2837c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Max) · critpt"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra","upstream_model_label":"GPT-6 Astra (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"66ba04e12aaf4e6691a98e0b8f593f0a22ffc7686de77bc7e1df5a71470251db","result":{"confidence_high":43.322603,"confidence_low":22.008593,"sample_count":70,"score":31.714286,"score_display":"31.7","source_stated_rank":null},"run_fingerprint":"f5f78536282ebab45bf3a99559994ab1f75738d18bb8dceeabff511169b52c51","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7ddf316df97abe576ae0257639ff57ff2baaf20910495002413b123a01c86a37","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · critpt"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-xhigh","upstream_model_label":"GPT-6 Astra (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c12e5ab064281cd6ccc1044f34c9ce60791ab1e946ccbd2af70840219d6a015a","result":{"confidence_high":43.027358,"confidence_low":21.762137,"sample_count":70,"score":31.428571,"score_display":"31.4","source_stated_rank":null},"run_fingerprint":"8204d4e039f0466bab6a65a0f4e3e447aed0f9a1dc9111c9664bff6cffa05423","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a3eab2ed185892c0656fd1a518e9eb03815090ab0b35259fdb81b0ae1345ad6b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (High) · critpt"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-high","upstream_model_label":"GPT-6 Astra (High)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7c3dc7540ad44e64d15a46d284a826e6836d26f8410378626f7dcdfcd5e6862f","result":{"confidence_high":40.350168,"confidence_low":19.564027,"sample_count":70,"score":28.857143,"score_display":"28.9","source_stated_rank":null},"run_fingerprint":"0a9fb0921a45d516be0765ccf75f02d82ccda17025e586fca865dadc062939f1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (High) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b78314a75b5364927e81e6d3fdedd719151fde37daa8672f0d119c102341d494","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Medium) · critpt"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-medium","upstream_model_label":"GPT-6 Astra (Medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"81901ef2da56c5a47e47f4f8158eba9faba269fe5d33b070a740dcb0e99db98b","result":{"confidence_high":40.64945,"confidence_low":19.806444,"sample_count":70,"score":29.142857,"score_display":"29.1","source_stated_rank":null},"run_fingerprint":"67b5ec14f3fd40104d889c9344dae56da1c22304718fba6a75da73f11c92c137","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e91c0610629df6f54fa1ec95c50c5390fbbb0e3cbd59abd96fd22467a6ff39d6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Low) · critpt"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-low","upstream_model_label":"GPT-6 Astra (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c5ea3b01050dd773498c0f7c18743b5879f13a02f8732e37b4e0187cc2292a80","result":{"confidence_high":37.635137,"confidence_low":17.403756,"sample_count":70,"score":26.285714,"score_display":"26.3","source_stated_rank":null},"run_fingerprint":"d49a99018f9a527a9f25330b717c329e912fa1f4ffa5e19b1075944b13ab9116","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T14:57:43.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T14:57:43.000Z","first_observed_at":"2026-09-03T22:48:46Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2dd3fc42ec786a9f884c2b0f3d53b8d071b9aeca5cb54c04444c953a97d0d65a","metric_details":{"best_score_across_scorers":"0.72","model_release_date":"2026-09-03","stderr":4.512608598542129},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"TAGZYToqgQmFZr7uYL4rSF","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5f6ca709f7db9440f480a20a35c6c44c23fd8b8ce80274cf37a49ff0efca86b4","result":{"confidence_high":80.84471285314257,"confidence_low":63.155287146857425,"sample_count":100,"score":72.0,"score_display":"72.0","source_stated_rank":1},"run_fingerprint":"dd8efdee8bfcc1af885108fa6fb2d01714bc74297921d78f45de07eae13f719a","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"272954374148a779b94cd2c80aeca385ce6122dd790529c6351a814d394dbac1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (Max)","source_effort":null,"source_model_version":"gpt-6-astra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d34c2d359536016f27e3aff27e6b7bc679d90fe39a26d13dbeb96ff4547f5eeb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"0ba6862f5cbeda972118c6b549a72a7abbbf38b7ecb2e6d11de68402669f786e","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"971660d462d4a7f9c705ca4380dfbd847f0a0c39a943e3fbd5f5d7742aa74e38","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (XHigh)","source_effort":null,"source_model_version":"gpt-6-astra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f8d008c01a0e18d71784e71cbeb2c3dd20d613c8d1bc9ddfa44675345ca9211d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":98.5,"score_display":"98.5","source_stated_rank":1},"run_fingerprint":"12f6f722993dc8c5e58ebfcde891f036b59dcac144818eeb70c8282efd93793b","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ce6ea4270d79fe1e7788fc74066043f990aec8ef2665af0e8d1603a9f3df5978","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (High)","source_effort":null,"source_model_version":"gpt-6-astra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a76d7c5ce089e416af0044a1b8265c1de1855953b35a050247efad55384d42b2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":98.5,"score_display":"98.5","source_stated_rank":1},"run_fingerprint":"f09e17c2cc665184887aa193462117036b2114c1b9a41be67fed535bf747a072","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"340a4f286426e8d8665d2c06398babab29e699dcd4e8c6ab4c16529b62e1ad6f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (Medium)","source_effort":null,"source_model_version":"gpt-6-astra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"cfeb8331faac7511001903567b60561f04867b8ceef1d993af7353fd9a74cfa0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"480d30093a56b34a1e6ee42a785444a72ce5c11059838e9d483a93312200ce00","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"58fb54e3960b9f6c1450cc7bc96b9b80f4309df31e47491c2c9570a186dae831","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (Low)","source_effort":null,"source_model_version":"gpt-6-astra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ef38246a3bf02554635cfa17587b22576e262667abef4be017c8589d5f309fe5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":96.5,"score_display":"96.5","source_stated_rank":21},"run_fingerprint":"534e414f965f818fc77e4d99bb89c8b4e0ada28f3e7eda5ede995c022e6186fb","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ff5e5ed1c3925254532e47454f1867326fe10ab97f6f008e91e07c7e30146b1e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (None)","source_effort":null,"source_model_version":"gpt-6-astra_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3f9d4631d9c1aa0509a0a11dcd0a32132d39bb93adb8255ab5c4acae3065dc35","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.0,"score_display":"86.0","source_stated_rank":99},"run_fingerprint":"32ec524c04b7b8ba60631e26b7e9cf1280b450c544a7dc88e02d6a0584e0f41d","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3807ea10a6fe1e9bbe8e157c","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-04T01:03:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T01:03:12Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"52da400c4513cb966dad2116a772e4a6a055a9abc03486fa0f2102253ae1ae18","metric_details":{"ci_attempted":452,"ci_half_points":0.8303197562056527,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":331,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":28.45353982300885,"mean_cache_tokens":1986676.7079646017,"mean_cost_usd":7.497827996681416,"mean_duration_seconds":1982.966814159292,"mean_input_tokens":2183033.681415929,"mean_output_tokens":61148.50663716814,"median_agent_steps":26.0,"median_cost_usd":6.712564,"median_duration_seconds":1773.5,"median_input_tokens":1767876.5,"median_output_tokens":58617.5,"median_output_tokens_to_pass":58206.0,"median_peak_context_tokens":null,"n_attempted":452,"n_passed":331,"n_tasks_attempted":113,"n_tasks_passed_any":90,"pass_at_4_points":79.64601769911505,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_6_astra_max","source_model_version":"gpt-6-astra","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"d4e26041d9b76d03a07c7366187d69f1095660ca79d9891f8caf0c00b728abb8","result":{"confidence_high":74.06040825178087,"confidence_low":72.39976873936956,"sample_count":452,"score":73.23008849557522,"score_display":"73.2","source_stated_rank":4},"run_fingerprint":"3b9fde1893a272dc3b59dce4eb0ab69a9fda40d39d87d5fcaed61eae87bde541","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-04T01:03:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T01:03:12Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"c5653d614979e33e1e190a43b26fc63602175a46d5a7494640ea18a03d849fd5","metric_details":{"ci_attempted":452,"ci_half_points":2.865396174075141,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":335,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":28.754424778761063,"mean_cache_tokens":1326921.6836283186,"mean_cost_usd":4.429117203539823,"mean_duration_seconds":1132.4004424778761,"mean_input_tokens":1456927.0929203539,"mean_output_tokens":29557.327433628318,"median_agent_steps":26.0,"median_cost_usd":3.8406789999999997,"median_duration_seconds":959.0,"median_input_tokens":1163918.5,"median_output_tokens":28542.5,"median_output_tokens_to_pass":28361.0,"median_peak_context_tokens":null,"n_attempted":452,"n_passed":335,"n_tasks_attempted":113,"n_tasks_passed_any":91,"pass_at_4_points":80.53097345132744,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_6_astra_xhigh","source_model_version":"gpt-6-astra","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"cd2c8b35814b3638135ca10c56b669d6445847ca677e4ff099395bb58a65aee7","result":{"confidence_high":76.98044042186275,"confidence_low":71.24964807371246,"sample_count":452,"score":74.11504424778761,"score_display":"74.1","source_stated_rank":1},"run_fingerprint":"e3a57375162520c87a106ecfc18be325042542c7b102dde54fd8ef4c67e836c6","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-04T01:03:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T01:03:12Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"163fbbe46502da0970e3e783082c4985339915336c5824140d97ea2df90396b5","metric_details":{"ci_attempted":452,"ci_half_points":3.423496057873005,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":331,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":27.424778761061948,"mean_cache_tokens":1168811.0420353983,"mean_cost_usd":3.9237237909292038,"mean_duration_seconds":1038.6858407079646,"mean_input_tokens":1283270.8030973452,"mean_output_tokens":26505.853982300883,"median_agent_steps":25.0,"median_cost_usd":3.454243,"median_duration_seconds":893.5,"median_input_tokens":995020.0,"median_output_tokens":25414.0,"median_output_tokens_to_pass":25170.0,"median_peak_context_tokens":null,"n_attempted":452,"n_passed":331,"n_tasks_attempted":113,"n_tasks_passed_any":93,"pass_at_4_points":82.30088495575221,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_6_astra_high","source_model_version":"gpt-6-astra","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"745bdfdb720bdb42019732e65651cafa660510a64f9f5185702998f284d008e8","result":{"confidence_high":76.65358455344823,"confidence_low":69.80659243770221,"sample_count":452,"score":73.23008849557522,"score_display":"73.2","source_stated_rank":4},"run_fingerprint":"f1452e16b5ccb4e9e74e1925e4fc8899f89e2087a44a4075d2c433fcacf19c04","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-04T01:03:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T01:03:12Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"fdfd325eb06241c4836fe2631dae420441a189c66f1f6a2cd7e4ef6bb9dbca71","metric_details":{"ci_attempted":452,"ci_half_points":2.5896490818318694,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":329,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":26.030973451327434,"mean_cache_tokens":916944.3473451327,"mean_cost_usd":3.0754742975663714,"mean_duration_seconds":883.9358407079646,"mean_input_tokens":1008238.2898230088,"mean_output_tokens":20361.809734513274,"median_agent_steps":24.0,"median_cost_usd":2.6204002500000003,"median_duration_seconds":747.5,"median_input_tokens":779601.0,"median_output_tokens":19006.0,"median_output_tokens_to_pass":19006.0,"median_peak_context_tokens":null,"n_attempted":452,"n_passed":329,"n_tasks_attempted":113,"n_tasks_passed_any":93,"pass_at_4_points":82.30088495575221,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_6_astra_medium","source_model_version":"gpt-6-astra","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"ef3740264d5d8f3a6af4435d8a23f6049cefd4cd0d64bd793a16f05067ce0f57","result":{"confidence_high":75.37725970130089,"confidence_low":70.19796153763716,"sample_count":452,"score":72.78761061946902,"score_display":"72.8","source_stated_rank":8},"run_fingerprint":"6a11c93595a3c64566e6e53f549675d325d3327e0af68ef857a08fc91519cfb4","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":31,"at":"2026-09-04T01:03:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T01:03:12Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"b7fd8cd1eedfc72d5d1e597b48a270b671a9ed32fda9e4261cd4c6c7c8cb1ae6","metric_details":{"ci_attempted":452,"ci_half_points":1.3008610516858732,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":303,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":19.537610619469028,"mean_cache_tokens":444727.3495575221,"mean_cost_usd":1.595223439159292,"mean_duration_seconds":613.4247787610619,"mean_input_tokens":494506.91150442476,"mean_output_tokens":10579.53982300885,"median_agent_steps":17.0,"median_cost_usd":1.26315875,"median_duration_seconds":480.5,"median_input_tokens":310994.0,"median_output_tokens":9487.5,"median_output_tokens_to_pass":9650.0,"median_peak_context_tokens":null,"n_attempted":452,"n_passed":303,"n_tasks_attempted":113,"n_tasks_passed_any":90,"pass_at_4_points":79.64601769911505,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_6_astra_low","source_model_version":"gpt-6-astra","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"43d663d80b591313b23744f5f1e3fb3d6ffd010aeb26bcaca1bb390dec495b0e","result":{"confidence_high":68.33625928177437,"confidence_low":65.73453717840262,"sample_count":452,"score":67.03539823008849,"score_display":"67.0","source_stated_rank":22},"run_fingerprint":"bfc6e080ac8cef80250f119fa2f6bdfacc6171486f5dc9094f1a516e1fff38f0","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":28,"at":"2026-09-06T19:02:21Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:21Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3403b5525f2e094f9cbd0c5d0899aa2784f50e87fc35fe9da90f8bebf90cc2ed","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"28.45353982300885","mean_cost_usd":"12.369032401327434","mean_output_tokens":"61148.50663716814","model_release_date":"2026-09-03","notes":null,"pass_4":"0.7964601769911505","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-6-astra (max)","source_effort":"max","source_model_version":"gpt-6-astra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"18855f7d70748eced758f51cfff39d44ab74e8a13e08a1959879db37bb8313cd","result":{"confidence_high":74.06040825178087,"confidence_low":72.39976873936956,"sample_count":113,"score":73.23008849557522,"score_display":"73.2","source_stated_rank":4},"run_fingerprint":"9757463c5d118e401113697862a4b6f9188b4290143492bbc9fa24f711a539e6","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":28,"at":"2026-09-06T19:02:21Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:21Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"506bc21482fa9e3a21e396b08bea2ca6589c1aa667c62d470c66b89c02800c04","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"28.754424778761063","mean_cost_usd":"6.52377356460177","mean_output_tokens":"29557.327433628318","model_release_date":"2026-09-03","notes":null,"pass_4":"0.8053097345132745","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-6-astra (xhigh)","source_effort":"xhigh","source_model_version":"gpt-6-astra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"06411a3c7b7a7adc49d95ccae81d5836a2d61d81e8c3234d24f1e12d65119a1e","result":{"confidence_high":76.98044042186275,"confidence_low":71.24964807371246,"sample_count":113,"score":74.11504424778761,"score_display":"74.1","source_stated_rank":1},"run_fingerprint":"0f72722250584b8ac9b661fd59e71a65ad06162632378ba831de54e9c8b22c58","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":28,"at":"2026-09-06T19:02:21Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:21Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6c002679c9d21b9ada750214a282e081565b08beb7906174f480e63ebb04ec64","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"27.424778761061948","mean_cost_usd":"5.723721956194691","mean_output_tokens":"26505.853982300883","model_release_date":"2026-09-03","notes":null,"pass_4":"0.8230088495575221","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-6-astra (high)","source_effort":"high","source_model_version":"gpt-6-astra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"315d5e4f090b2c6568ba693640b0da02780e5c2d2d81bf628c1dc840927484ac","result":{"confidence_high":76.65358455344823,"confidence_low":69.80659243770221,"sample_count":113,"score":73.23008849557522,"score_display":"73.2","source_stated_rank":4},"run_fingerprint":"d650c36906b253b281068669ad07852613f619ba7b510e93dbf87fd908ed2aaa","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":28,"at":"2026-09-06T19:02:21Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:21Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"019b0db77ac76e4215b8f9a4fdfc6c475f64a1828f10e73b587ef4af6d5b262a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"26.030973451327434","mean_cost_usd":"4.379527705752213","mean_output_tokens":"20361.809734513274","model_release_date":"2026-09-03","notes":null,"pass_4":"0.8230088495575221","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-6-astra (medium)","source_effort":"medium","source_model_version":"gpt-6-astra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"744ddb141669201f57c9f56757bc2fceebec8429d24327b03ea9ff03a18c8622","result":{"confidence_high":75.37725970130089,"confidence_low":70.19796153763716,"sample_count":113,"score":72.78761061946902,"score_display":"72.8","source_stated_rank":8},"run_fingerprint":"f2e25c757f8765eaf62d84684520ea352e4264d45b6b94259d03b604fdf96954","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":28,"at":"2026-09-06T19:02:21Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:21Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"69553a655111fa07f07757a0994ab3f761e55d9a950081c4d3f375ad025911fb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"19.537610619469028","mean_cost_usd":"2.188762706637168","mean_output_tokens":"10579.53982300885","model_release_date":"2026-09-03","notes":null,"pass_4":"0.7964601769911505","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-6-astra (low)","source_effort":"low","source_model_version":"gpt-6-astra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"bc348d60d446f55db6b212bd1ced60d90db90bfc70e3bf08a113b36a7557bcd3","result":{"confidence_high":68.33625928177436,"confidence_low":65.73453717840262,"sample_count":113,"score":67.03539823008849,"score_display":"67.0","source_stated_rank":22},"run_fingerprint":"63d1ecc28e74f9ceaf59afacf5039202d0fe36aef86addb9d7f890f9472a7a0c","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:21Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:21Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2896edac801ccbd9a120206679ac3d527c3be0d607e24b40eb5aa1574d962ae3","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra","source_effort":"max","source_model_version":"gpt-6-astra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"codex","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"e19b193e393c9e81b783925a453da14203f8cff7a54b1f5af520cac4f7770b2a","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":53.26,"score_display":"53.3","source_stated_rank":4},"run_fingerprint":"b7b3d41863bf0565992a9afe19aebe52d3b68f1a7081c37bb4c2ec16479ca6d5","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e7584ed4a6cb0836c069c7510a664c82d67a6dee96c0d162769df9ccb89be903","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":6.495051,"date_is_proxy":true,"latency_seconds":1899.783,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.0},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6-astra"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"32daa5ecbf93918753f010feda35d364d4743fba828053f7e9dd330b259574f5","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":null,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"8d7315c56ab6b55c86aa997ef19b50ec274bbbeb6929effba01284cb1fe69f11","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f19a71b55802cfdd9970430695f8bec008cec13cdf25f8b285904adac511f8ea","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (max)","source_effort":null,"source_model_version":"gpt-6-astra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"23434efb8ddf493bc48d90631ec21649f317cf1fa40a5bf6af29c124520e9bd9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.4814814814815,"score_display":"56.5","source_stated_rank":28},"run_fingerprint":"389e3324e35482c7a56720e13c5c4adfee8cd0820f7993963639754e90c66dd1","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"df6576b146468ee37b02b2c0f046a5c306f7ee982bc7db79b0c07d9b790e4e2b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (xhigh)","source_effort":null,"source_model_version":"gpt-6-astra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"be5aa1f7f5ee4a0c47bcc9b271ca2e6ac7e728d6f6628ff92a6df34d61e84b3a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.671296296296305,"score_display":"55.7","source_stated_rank":37},"run_fingerprint":"5657ae3756687065cb0bf4ece55fdadc30a379a7f8a1af6b2fe8a95ebc3ce746","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b83ed5c62603267713de452846a7774bcbf952d8906473c5dded6dc31b00653c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (high)","source_effort":null,"source_model_version":"gpt-6-astra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"170a3a512faec30a692d6f610460dae876d45cc1d800a5fa7f85d494fee4b46c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.4398148148148,"score_display":"55.4","source_stated_rank":40},"run_fingerprint":"a8227e344abf8bf7db58a83298908f358fadfecf5efe61808412a2a1e7f01b4e","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ec6f07ed8ada7a01c9cfa17d3e6c86899f16aa6fbfadd87a0e3de8362f85bf8e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (medium)","source_effort":null,"source_model_version":"gpt-6-astra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"9d71ed439ea4d45d09cf63f1c4c5d45222641cacab99ea15b91f7ef61f2fdd4e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.1666666666667,"score_display":"54.2","source_stated_rank":53},"run_fingerprint":"3e3793611c6fb254da3211741930e61e6021003966d68775eb2cf4ff1d329e35","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"06d6a57e132e6d58c0045a65dcb27ff078d5045973846a9030ac34621691aebb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (low)","source_effort":null,"source_model_version":"gpt-6-astra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fff2cac787e429a8952198bf513c3edc166c75859007bce4302afbf483a80c78","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.050925925925895,"score_display":"54.1","source_stated_rank":55},"run_fingerprint":"1ea6e88bd2d99019a8c39de539f682600e2925da92d8d39dc9d7cf02b89e1d3b","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b8c6909976149019c565e2c4ae128a4a74964542a64671a10ac65dea19c15028","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra (Non-reasoning)","source_effort":null,"source_model_version":"gpt-6-astra_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fef6da920f63af1855b6340bc76778d764a891ba3ddbe7baf0db12c0cbe24777","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.4722222222222,"score_display":"53.5","source_stated_rank":62},"run_fingerprint":"a8888babc5b404f255a00c4a71b595dc65906280da2919f4d0757cc6913a3e8c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3807ea10a6fe1e9bbe8e157c","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"895b415a144d6c9762234cebde81effe0fe61ccfaea88bcace58bf2169d0670f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Max) · scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra","upstream_model_label":"GPT-6 Astra (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2c1609f94d68b5b5b4e4544345129d270fc9103d626986e311c94147671d37b1","result":{"confidence_high":62.084973,"confidence_low":50.707354,"sample_count":288,"score":56.481481,"score_display":"56.5","source_stated_rank":null},"run_fingerprint":"77ca3fd1b5e65e0368e8bdf0b3357eb52f236b937338e9c372bbb9bf79092f2c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c12726f6397f56a29428e4822c37fea343065f9dc7bd2ec9ee59b7f85e257f26","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-xhigh","upstream_model_label":"GPT-6 Astra (Xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"cecc9a394bc5d1dd989a7f0c1303e2a423ad2de363abd24d24431b5b081681df","result":{"confidence_high":61.296683,"confidence_low":49.896603,"sample_count":288,"score":55.671296,"score_display":"55.7","source_stated_rank":null},"run_fingerprint":"4764fe0373ee2f29f8924d6e3fdda4374dd11bffe589d68385854d0d72d51e1d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ba0e401f443774086370af4d3ed6a82830b965c0c52d4cd2331cabdf2c364bdb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (High) · scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-high","upstream_model_label":"GPT-6 Astra (High)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"13a649407f94578629c218fd9d71103da8fbc8e9aab781616bf9e0090af2dcd3","result":{"confidence_high":61.071179,"confidence_low":49.665239,"sample_count":288,"score":55.439815,"score_display":"55.4","source_stated_rank":null},"run_fingerprint":"4ddd3bbdf61b3c28dd450c7d72757846c75077b820459099bf643cc54d29c3e5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (High) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f8d7ca619091b882ddea4cb90304d5567566ecc45bc2d87618693bc4866e5688","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Medium) · scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-medium","upstream_model_label":"GPT-6 Astra (Medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"bb856a07bb547bdbb28decb238b4660e1caa545715f3e6f84c32baa27fb1e4a6","result":{"confidence_high":59.828702,"confidence_low":48.394938,"sample_count":288,"score":54.166667,"score_display":"54.2","source_stated_rank":null},"run_fingerprint":"16d4497ad0dbd459fa520477804414b4e82ffd0c414cb28d62923d62437dd6d1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4558596ad90e2bae6db00fa7dad0e29e10931ef891d8126ad1e7821db5691739","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Low) · scicode"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-low","upstream_model_label":"GPT-6 Astra (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ad8df35a5c2c0b4d9bce701f15f4cd68955836957997b81e1722d2383a49d36d","result":{"confidence_high":59.715565,"confidence_low":48.27964,"sample_count":288,"score":54.050926,"score_display":"54.1","source_stated_rank":null},"run_fingerprint":"3221168dd42efe2f4083839eea70f86851c59abf3f51eb08a6a75e22884d9a26","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f0237e2cb3677013a6df14e75ca54a2996666ad4fa048ec8a488e77f07d64b3c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"2.86415","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-6-astra-max","source_effort":null,"source_model_version":"gpt-6-astra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"c2edd8d4c38660c681e4fd4857862e563cc9344321c9f62a8287a3235d44a90c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2951.32,"score_display":"2951.32","source_stated_rank":1},"run_fingerprint":"2a35016c89f8c9293f34630f0afd8090e2f251177ad0a3d8b2caa123fe83dda0","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2abeb9e8ecd53f9507e63389c99f1f6f1567dd1a77b979793be3fb65fe07ee44","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":38.512323,"date_is_proxy":true,"latency_seconds":2344.127,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.174},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6-astra"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"c5ab89d9df043da49b5f7e87fb2be9e6e2cf4d57dd0c771d807f112a7ea852cf","result":{"confidence_high":93.85503999999999,"confidence_low":85.33296,"sample_count":null,"score":89.594,"score_display":"89.6","source_stated_rank":6},"run_fingerprint":"ae572b2bcca7c4408cd183c4a9df782ada4175ccd8ec1e50550051480ec24a78","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ada290ab84c7c93f7941081b5eb7400a169ab65e692900f3d2b7dbe2f051bca6","metric_details":{"license":"Proprietary","variance":27.537801047031785},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-6-astra-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"95b23b2bc224b04a71275c396133cdc2f14d39874792d71de27ed62e33a2870c","result":{"confidence_high":1797.9511046440557,"confidence_low":1777.380705646016,"sample_count":6123,"score":1787.6659051450358,"score_display":"1788","source_stated_rank":2},"run_fingerprint":"9bfbb8b4938b18805213db5f7fb1ca7cd5b6077da7aac8d2b6d620039b8ffabd","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9c572a3c20deb33e61e7db09d712a0fba4610202099ab4e91045497760e03b52","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-6-astra_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"a7c1ca5892f481ac57ea5914092adc0c69d1c4d086da59fc6905e67fdfc8dce9","result":{"confidence_high":1816.76,"confidence_low":1783.81,"sample_count":2281,"score":1800.28,"score_display":"1800.28","source_stated_rank":2},"run_fingerprint":"abebc7a7d3398f75d23c1ba67a943da9dce031f342f5a901f94bc35001ed530b","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"mirrorcode","metric":"mean_score","name":"MirrorCode","release_id":"release_07bf24088c0805f7b6df7a88","split_or_window":"15-programs-30-language-tasks","unit":"percent","version":"ML +Private 2L"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T16:51:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T16:51:09.000Z","first_observed_at":"2026-09-03T22:47:34Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"eac9647bcc5d07d2ea4d796e53d1663e7a897229d3127801c11ada04fa9ade20","metric_details":{"best_score_across_scorers":"0.467","model_release_date":"2026-09-03","stderr":11.86},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mirrorcode-ml-2l-gpt-6-astra","log_viewer":"","logs":""},"run_count":3,"scaffold":"Epoch AI Inspect ReAct agent","tools_allowed":"shell, text editor and visible-test evaluator; no internet"},"protocol_fingerprint":"6e1701124b46ba8be1fdf58cc4ee5606be9333007eeab76e3f8c894f428e0de9","result":{"confidence_high":69.9456,"confidence_low":23.454400000000003,"sample_count":30,"score":46.7,"score_display":"46.7","source_stated_rank":4},"run_fingerprint":"e7283776d09df0d07850762cab5a4a4ada316cb65c80593dcb5b6445dd4e3e1e","source":{"content_hash":"3e212e97e03d0343e676ecc7ff046ee7b94d219ba7d863364634b7361dc89b4b","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://epoch.ai/benchmarks/mirrorcode","id":"epoch-mirrorcode","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · MirrorCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/mirrorcode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e27d13c28f19e048161d0060a2a1073074e0312b8252629f434c7891800ab72b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"service_tier":"pro","source_display_name":null,"source_effort":null,"source_model_version":"gpt-6-astra_promax","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"8ef8215c0ebf37651b14f2e485100d791f62dd592f8b348bbe2f92dd1dc5c1cd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.57,"score_display":"93.6","source_stated_rank":1},"run_fingerprint":"45759a5d4a0028f284e3dc4d0318bc59dce3feb835ddaa23a53e13833e1fae1e","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b6f7afbc908e06af71c12db2fb587375a49f3f1faf3b255c289633461fd3fbee","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-6-astra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"9f9b8601173ca2a58dbf30bdcef804dfc38480071cca5d16768a912729f0d59b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.28,"score_display":"93.3","source_stated_rank":2},"run_fingerprint":"237f67e5985c29bd228cc698494b000ce11ef1cd074d68b412e6702d2a0200f0","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7d3460ae3d9168fcc9935da69b9504549e849102b141e7ed289f213912cbe97c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-6-astra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"55d52723b5c30de4f3f8867328258eefec6ceae8677d39f60e1d304ee4f07bce","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.87,"score_display":"92.9","source_stated_rank":4},"run_fingerprint":"6ef4ff4704b7ff26b9e656bca78a7770ea474a08d1e229e65d8684919a0ec35d","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"22da4fe22f1389980532eefe1932d5d2c48c5905dcfb371882f5f7105e02fc65","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.339493,"date_is_proxy":true,"latency_seconds":274.44,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.375},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6-astra"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"0d1dff87a8f24d56430cfa8ec4d1451cfab111d3ab7356a6e89ce331a3683569","result":{"confidence_high":88.001,"confidence_low":86.531,"sample_count":null,"score":87.266,"score_display":"87.3","source_stated_rank":2},"run_fingerprint":"9bf86a024bbf838fda54793b79291bbf9972cd327a92ec0764eb9fa62bb93ced","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-03T22:50:26Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:50:26Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e92508a05bd908b87149c95d7e32186fd076c9c71522b57832b1a3b8e1902b88","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Astra (max) · terminalbenchV21"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-6-astra","upstream_model_label":"GPT-6 Astra (max)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"32855f9edc530f66a1ef70a99ee367abb7801b98a0ca878415384b9c12400512","result":{"confidence_high":93.508285,"confidence_low":80.093778,"sample_count":89,"score":88.389513,"score_display":"88.4","source_stated_rank":null},"run_fingerprint":"3bf41ecc927a1faba0d40bef1eae4057301a42712fb3d635e5cafc219de24bbd","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (max) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-03T22:50:26Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:50:26Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b503fc17a27abbd1b004c1e4de477d3b99b18da875bb60c5b3718f6cb0c203fb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Astra (xhigh) · terminalbenchV21"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-6-astra-xhigh","upstream_model_label":"GPT-6 Astra (xhigh)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"fdf3b1ca1b416989c95319ec107d2a580bb09e6a54ea4a80c8e302d1668474f1","result":{"confidence_high":94.052379,"confidence_low":80.985823,"sample_count":89,"score":89.138577,"score_display":"89.1","source_stated_rank":null},"run_fingerprint":"409261aefdaad250c5a457d1fcc963fb3d290ffef646b90e1f8f2e3a029d00a0","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (xhigh) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-03T22:50:26Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:50:26Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a2185521a372ea043a08dc956725ac60755534a17b1252bf24b0e6ccb3328dd3","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Astra (high) · terminalbenchV21"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-6-astra-high","upstream_model_label":"GPT-6 Astra (high)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"e2931b56e427e7a6a42f489c8f02b2331d34f6954d0e1cb776b1dcc9b7a6dc7d","result":{"confidence_high":94.588206,"confidence_low":81.886132,"sample_count":89,"score":89.88764,"score_display":"89.9","source_stated_rank":null},"run_fingerprint":"fbf0821b8393be7e23569d06e44bfa5f89cfdbec18d6180757d5a61ab8928ff9","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (high) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-03T22:50:26Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:50:26Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"58633f3d2cc976247348be036a6fb487659016cfcf0b7a8367fcab6ee9e8100b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Astra (medium) · terminalbenchV21"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-6-astra-medium","upstream_model_label":"GPT-6 Astra (medium)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"bfd92362cb329a06f07f4f64282e25e762a2d58a0a02f147698c4afe644c710a","result":{"confidence_high":94.321369,"confidence_low":81.434902,"sample_count":89,"score":89.513109,"score_display":"89.5","source_stated_rank":null},"run_fingerprint":"91be0f171965637071272317e475b9062d9451e778b98dfb65ea391486af6c60","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (medium) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-03T22:50:26Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T22:50:26Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"89190b35d599053a52a488237bf9a824a725e056c72973d40322a352bbdd2f7b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Astra (low) · terminalbenchV21"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-6-astra-low","upstream_model_label":"GPT-6 Astra (low)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"3b7542763248a0fabf6bc273a3261533fd9f8a5744c6c906fa4ff76d2dd03478","result":{"confidence_high":93.233339,"confidence_low":79.650655,"sample_count":89,"score":88.014981,"score_display":"88.0","source_stated_rank":null},"run_fingerprint":"f91b5993dd301ac0f755f6f086102f31bc2fee014659209123e743febc2a4e5b","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (low) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2cdf5e0fc5a289d88122eb397f80b43a92da8655e2b35ce4e8c584b4ad4e8231","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Max) · tauBanking"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra","upstream_model_label":"GPT-6 Astra (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"b5fefcefe4c253976a19054e4400c31c48b17062f0e3d480f01926c742a8cb84","result":{"confidence_high":51.389846,"confidence_low":32.148693,"sample_count":97,"score":41.443299,"score_display":"41.4","source_stated_rank":null},"run_fingerprint":"017e9231868831f89016d3da4bc53e02465e3c2203bd59c9049501dbca8a24b1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Max) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ac4d8c4137569bba828b3deb4d83ce5b55987c802f63bd9c271782dd2d571ed8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · tauBanking"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-xhigh","upstream_model_label":"GPT-6 Astra (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"bf08c9d963ee2afff531f0907b0c61a7308be6ec35dac64a1496a10cb0e1f42b","result":{"confidence_high":53.02495,"confidence_low":33.686884,"sample_count":97,"score":43.092784,"score_display":"43.1","source_stated_rank":null},"run_fingerprint":"4c449b5c11321b7fcded4ea2566afdbcc7a6615172607cc998a25bd02a49ae5b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4c26be46953452e0e142aaa39d85550ddd202acbdf6d92554d79bec715c7433a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (High) · tauBanking"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-high","upstream_model_label":"GPT-6 Astra (High)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"25764cd39a9ad6d8be4910549b8dc25d6fc78e06280aa753443251e4cfd70ddb","result":{"confidence_high":49.950388,"confidence_low":30.81152,"sample_count":97,"score":40.0,"score_display":"40.0","source_stated_rank":null},"run_fingerprint":"05f9dd67958c304cfc68b411ed9f2e827d19e528fb7c414f25692244890ff489","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (High) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e2fce5b5e283519ef8bc245daff1dbb435f20099a916e13b97800f554e181779","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Medium) · tauBanking"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-medium","upstream_model_label":"GPT-6 Astra (Medium)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"7ec83161c1c1ff2841f113d0724bd5945d8812c88ffc8683933c1edf6f9958c3","result":{"confidence_high":45.371587,"confidence_low":26.663765,"sample_count":97,"score":35.463918,"score_display":"35.5","source_stated_rank":null},"run_fingerprint":"05facab800553c798d88815e1121e0a7d846d4aad5c6f151f5e2ce865efdfe71","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Medium) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4c4b37474e299fc529887fde1ef2f8f987c5c344ed102fd10b26219ca32fcb50","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","source_locator":"Leaderboard models table · GPT-6 Astra (Low) · tauBanking"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-astra-low","upstream_model_label":"GPT-6 Astra (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"ba132022f40cad94ce237a8dd062f4e7491cf7f9371fd124b41d102bec299a8c","result":{"confidence_high":41.773554,"confidence_low":23.518548,"sample_count":97,"score":31.958763,"score_display":"32.0","source_stated_rank":null},"run_fingerprint":"a03dba05149dcb5aa56ee7982fec9b9159d9fa48de35f283bd9337cacb8bc304","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Low) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"001d549af87597a95f9b666a6d670e68edae8860fc549c9897bd56a78c738aa9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.751","mean_standard_error":"4.6","model_release_date":"2026-09-03","notes":null,"standard_error_points":560.0,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra","source_effort":null,"source_model_version":"gpt-6-astra_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"bdd5aa2fdf1110e0350fbd2dc4d16fac50c18134b907ca69fcaebcdd11f0252f","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":64.7,"score_display":"64.7","source_stated_rank":7},"run_fingerprint":"f8dc376bc0c4ba49322c78743912f750c4f26d0a67622f38753dce5d136dbed5","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_138452291adbe4025c01d25b","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2e6c3ccf9ffe4c1e5d7c146b3b63496d7235636368d21646f635073be18277dd","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":11717},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 6 Astra (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a5ae9815f95f200f6e7fc8f05e874ccce025a7480673d3be28a29dd1ff6db162","result":{"confidence_high":0.14501263879072898,"confidence_low":0.100413035461178,"sample_count":888641,"score":0.12271283712595349,"score_display":"0.1227","source_stated_rank":4},"run_fingerprint":"b22cf20515884ea7838e501065a933110e03f39445468d685d77418014263c28","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"77a80a05a39a13928414489d077319d00e5c97b4cf936d0367bd99ecc3f82fc2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-03","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Astra","source_effort":null,"source_model_version":"gpt-6-astra_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"cc9fb18a90926b80745487407023b43b6b38c6beb0a76f23d2522810152ae557","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":15514.701666666668,"score_display":"15514.70","source_stated_rank":1},"run_fingerprint":"3a442d92722619ea1bf2c4e0381268c6d63be2a0e04de90a356185e9f83fbfea","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2315d6296613da07b43f9c55","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3eebc6748bf9c6db10fc64507927a61c47642f01fe44cd8d00008a10b04fc7ad","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · GPT-6 Astra (Max) · gdpval"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-astra","upstream_model_label":"GPT-6 Astra (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"48b4cfc9bee30943438b60d8a4613209c4a6747a28e3f9f27b6df2614ff9803d","result":{"confidence_high":1567.2,"confidence_low":1516.58,"sample_count":220,"score":1541.89,"score_display":"1542","source_stated_rank":null},"run_fingerprint":"9dcfc0691e9e63778cf83f387de8952da462c664b03a305a6f4186cac9cae694","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a45ce5352161ba5d3a87a651","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"744ab8f2960e1bca0357982ebd9f0036e9141678f6f1261457866d5424d83f96","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · gdpval"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-astra-xhigh","upstream_model_label":"GPT-6 Astra (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"57dad060d1dc7ee2bea3dacc9d6b776b93e1ee3b6d8c0aa3d983f3705a7bb999","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1516.48,"score_display":"1516","source_stated_rank":null},"run_fingerprint":"f4ea430c01725651d0c29551a656e45ed08c3ec8616ded437338865d6e7087be","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_68496eb7a3dbdd992be334df","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"98a2346382d8d3ace7756fa80ca9b165e84a3308c18c691c1c121f9a2cf82b89","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · GPT-6 Astra (High) · gdpval"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-astra-high","upstream_model_label":"GPT-6 Astra (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"3791972585a8a5b1fbd4e735043c8979921bbf76216fa59d0f7cd9c94e759c41","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1484.52,"score_display":"1485","source_stated_rank":null},"run_fingerprint":"f0ae88ce6719a75889985566592d97a6edb18b55f230c5ab2bb7ff0ed7512216","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8eb1aa90528f25d7a199e22","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2918601cbc90802b483494241abfdc55af64092c6c8ce60ed1005b842e0737d2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-6 Astra (Medium) · gdpval"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-astra-medium","upstream_model_label":"GPT-6 Astra (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"035859dc70e1c38178f77ff44be55733444b48e0ad49f7ad0695f61148014c5a","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1467.88,"score_display":"1468","source_stated_rank":null},"run_fingerprint":"b08474ad2539d4fe35002da65f9cf3482e9152e7aa8f0cf60a2c458ef68d79f1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e68f34c626914bc40542bf8a","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eed50123bcade7014ed97f6f00b9ca523b8b7687e59e83ae9973249b099ec709","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-03","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · GPT-6 Astra (Low) · gdpval"},"model":{"id":"openai/gpt-6-astra","name":"GPT 6 astra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-astra-low","upstream_model_label":"GPT-6 Astra (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"5d523f902590be9f1b006110ad134c02f3072c7eb58483647eddc7fecf0301c9","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1365.6,"score_display":"1366","source_stated_rank":null},"run_fingerprint":"783bda8490b63a47f9265ec5b6d7f7b789059a6662b9abec1c8d0f22e08e7c0d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Astra (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_232c021fb22075f21dfc7e04","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7db8e447218f52c53403dac9c74780755e0c25c52e47151c7652010e22b85df8","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-09-28"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Sonnet 5.5; model release: 2026-09-28","source_accessibility":"API access","source_model_name":"Claude Sonnet 5.5","source_model_release_date":"2026-09-28","source_model_versions":["claude-sonnet-5-5_max","claude-sonnet-5-5_xhigh"],"source_reasoning_efforts":["xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"54f32cd2c2ca865d5447ced1a6a5a0ec09d02957d892eec34b0b3e1571ad4245","result":{"confidence_high":169.39,"confidence_low":161.81,"sample_count":null,"score":165.2,"score_display":"165.20","source_stated_rank":3},"run_fingerprint":"10e08d748aaf38d8fd86e1947a2908aca5057c512f9e64f86185f689d82577bc","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_79445b70144959c79964f01f","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ba272ed2395e2df5bfc394f7ad67f5d2869c5621b97ed1ccf61e77a5deabb5a3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"f181e808ea9805dbb1993e30735445e8f74a8fe7dd6a22fd27edc655c6d1473b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.000129,"score_display":"56.0","source_stated_rank":null},"run_fingerprint":"439d1d98c1ac00390aff41d5ea385144d023c735d9ec605b9ca3dccd713931f4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2b051ed238ac0d39a623255d40cdd9a668e4aa9467cc6e1b31b6c2c41675d6ca","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-xhigh","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"511613ac6765c046fcfd4bd7ebbec0722a8f0b22260da7f092b5ea685662fbbc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.895554,"score_display":"51.9","source_stated_rank":null},"run_fingerprint":"a95cb30a1d25bc2825651cf40f8e391ecbeb1c931df3e8a3dcfe97c65b56fda7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"20cd415233b26db8247560151866ed0079a71f7e4edbd6d07aa9eee1be025f0c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-high","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"ba94f65dccf4109e80b1c4d4be4a6a0e373fce6f9377b5f4a79da4e0a78dc073","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.753353,"score_display":"46.8","source_stated_rank":null},"run_fingerprint":"e66e6c0bcbe046d3a05868645d6dec1e687bb19f121cb685d2368be3ea242301","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7c63af9f64d8f76888dfa32873236898d25a17488fb0d1cd7d17abdb4d5abda1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-medium","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"c0a662ddd865423b0fd6d7aef7a506e117a0bcb110e291e32a7a441456c78be5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.838652,"score_display":"40.8","source_stated_rank":null},"run_fingerprint":"dc7321f944e6f0e8a66cadd47ef874aeaa39116ba91200bf193f6bfd67334a34","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_41b89fb2ca8e99b100b47ec0","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3fe4c5c8ead8d77af9c75b2ed517342d0fa659b853203afd316cb6f8148e2da6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-low","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"bf0b191bd88d63deca81535f39f3d3e2f3443210a5e3e4bad5225589a04d5215","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.868159,"score_display":"35.9","source_stated_rank":null},"run_fingerprint":"5c7eb93646844513a985f64a4f3b6d6e581ff8d3421ddfe638c394bdd85fc439","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_76ec00376e53620e063e0f5a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"094845b2c444e8ff62b4d400c187309124695b0c90f3589111ee95f391fa7f65","metric_details":{"license":"Proprietary","variance":28.323638143509708},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1481.4422482299615,"confidence_low":1460.580408466361,"sample_count":3145,"score":1471.0113283481612,"score_display":"1471","source_stated_rank":45},"run_fingerprint":"51adeb50c3e1758f0840864e3d92c9d090b2dacd1496d42efde5516cb0162c95","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4308b8e89da06dd80db0a25f45b8c4a528b1f3a87685d3be8107233c0431c77b","metric_details":{"category_scores":{"Agentic Coding":56.31333333333333,"Coding":91.366,"Data Analysis":59.48766666666668,"IF":56.78775,"Language":77.984,"Mathematics":96.13175,"Reasoning":91.6345}},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.67214285714284,"score_display":"75.67","source_stated_rank":36},"run_fingerprint":"cd96fd0c6165132be00e765c580aeb58ae8682e9e2b51813b4ed8b678ec22e61","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e5bbfd0cef35256b66d1e849a2b4b372f2a9744bc72be54f0e3e6305eabbeda9","metric_details":{"category_scores":{"Agentic Coding":39.34366666666667,"Coding":88.87049999999999,"Data Analysis":78.62,"IF":70.51650000000001,"Language":83.42966666666666,"Mathematics":96.69475,"Reasoning":86.80775}},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.75469047619049,"score_display":"77.75","source_stated_rank":22},"run_fingerprint":"c6d882a4963062ccf31b78fe183474d7ed6d27414e9a82fda342bf826ff035a1","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"966730d3540a3d4d00ecfc994917688db2ff81b5a6fe167077f3d7445c9c9ec5","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"266553aa78f0d034dc28936fc5b2d93b19ddbef89c4fcfdf37136ad31ec3b4a1","result":{"confidence_high":57.046839,"confidence_low":52.852129,"sample_count":2158,"score":54.958295,"score_display":"55.0","source_stated_rank":null},"run_fingerprint":"64ab398e2695ff844dcbbcccf40947f9a100b6f5d5630bd0d8695ff524803889","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d15d0384bfe29fc6c156835e2e697557a994a569d1c9a7d018cb7007ba3de354","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-xhigh","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"875d8b4b14969a7e5ae8e1f1148d47a6daf9b450b64d5f42236918840b7d180e","result":{"confidence_high":52.107726,"confidence_low":47.892274,"sample_count":2158,"score":50.0,"score_display":"50.0","source_stated_rank":null},"run_fingerprint":"112bd4ec4df42a2c49aa0a3ff721adde9fa899d8f0dfccfb503f14736367a784","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f936228627380ab110f81c488ba3a7e1a0e7d98c8a1e6da19574665c8d1f931f","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-high","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fc570b1a36f457ae2b92dc0e28e0863d3930d12a837c2a652d8b51a5d9990776","result":{"confidence_high":47.937277,"confidence_low":43.736489,"sample_count":2158,"score":45.829472,"score_display":"45.8","source_stated_rank":null},"run_fingerprint":"2bd37ed70291845fa8073a5fd59e30207081de0a5fac29e4c6f754a20269d139","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"531ab453cc1b3857575f462c3a44f5055ff9d73e430f7cfcdfe456be534a66d3","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-medium","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"32dab8987a33ff453cf1411a6c8974bdc4872f3bb75e351d14bf8ae3e84ae519","result":{"confidence_high":41.88702,"confidence_low":37.759962,"sample_count":2158,"score":39.805375,"score_display":"39.8","source_stated_rank":null},"run_fingerprint":"d32160ddfd38da4421c10a4d97f33f5dfd7863b2ce8611b94c7a98676190862c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_41b89fb2ca8e99b100b47ec0","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f890c585f91c4c3c2d50b62afc54473414e46cdd61e71adbf8f6649fbeaf844d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-low","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"3cad774f0ad0f813f9372aa50a9b72a9119d3cd1b75f31150ef407aec912ad2f","result":{"confidence_high":38.288168,"confidence_low":34.235259,"sample_count":2158,"score":36.237257,"score_display":"36.2","source_stated_rank":null},"run_fingerprint":"9a90de2161033e9bd6b042fdbc0f0bded54c3b36aed90f36303f0541c4c2a003","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_76ec00376e53620e063e0f5a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":5,"at":"2026-09-29T15:15:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-29T15:15:14.000Z","first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2e9c8566f8e582198232d866646177eaf404f2a4f54191f7f292f2064e49581b","metric_details":{"best_score_across_scorers":"0.465","model_release_date":"2026-09-28","stderr":1.5780495050030086},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"GtJ5yd3ZPQ7HVcm6Q9ssQz","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"24eec68deaf023f9661f8f974a8fe5deb3efb84f49f19ed44801de89e1b56910","result":{"confidence_high":49.592977029805894,"confidence_low":43.407022970194106,"sample_count":1000,"score":46.5,"score_display":"46.5","source_stated_rank":35},"run_fingerprint":"f67116d7d1e092b42351785012a5f4bc98160abedaa30b06ef27c053478f6cd1","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T19:15:12.163264Z","basis":"evaluation_at","evaluated_at":"2026-09-30T19:15:12.163264Z","first_observed_at":"2026-10-01T18:20:57Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8f477bb3c88c1775af246f58381feb330a408686a00f3f7f436dc5af90c3bd8e","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-sonnet-5-5-default","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5b6be51f131d2ccb3b5173bb09569690c2c134645d59d43788396e837cdac1a7","result":{"confidence_high":47.96350568874157,"confidence_low":41.53321203554727,"sample_count":null,"score":44.74835886214442,"score_display":"44.7","source_stated_rank":14},"run_fingerprint":"1ebb4a0c5aeb0d97ddb7c1ec4bc28528fe4e2603e66093cc0a3fd88afb6b0e01","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a73d17fe91f3e0695b858e89","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":5,"at":"2026-09-29T15:15:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-29T15:15:14.000Z","first_observed_at":"2026-09-30T18:53:22Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"98134f100faa5da25694dca5487839fe6ee3cf87cbacd0454e617d0471fb48b0","metric_details":{"best_score_across_scorers":"0.9558080808080808","model_release_date":"2026-09-28","stderr":1.3710320714521202},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"QPk8jbJvo4986sbWtdte6J","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"4a1225d698508261ad8c8935af5017f4596cb88e40c0d3222b44ae5f62fe27ed","result":{"confidence_high":98.26803094085425,"confidence_low":92.89358522076192,"sample_count":null,"score":95.58080808080808,"score_display":"95.6","source_stated_rank":2},"run_fingerprint":"3aee311c26787546158277ee6afd62c86c1fe6a62d5c2f8ec53cc711b1a42040","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":5,"at":"2026-09-29T15:15:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-29T15:15:14.000Z","first_observed_at":"2026-09-30T18:53:12Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"98624585922fdf75ca795b4ff540b474126390a2d8d7ae2e9e7f2e040abbe0e8","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-09-28","stderr":0.0},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bSEaZLPehiNVDQL7HP9EJS","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"edce3ee4cd626f35a9389beb93d9db7f14028e8cd382a514775b7b542e2944e5","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"9fbe934542883a3c11bd04322d5eb919497135a2b8a8bb9015b1032b649f8365","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":5,"at":"2026-09-29T15:15:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-29T15:15:14.000Z","first_observed_at":"2026-09-30T18:53:13Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"167ab50467890ac6c50a4d122c40785c3d1eb1c26a1b8aca2de8890b733a6a19","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.887719298245614","leaderboard_private_problem_count":285,"model_release_date":"2026-09-28","public_example_count":10,"stderr":1.8734030360524248},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"4tZWwV9tqEgsCaKeSuSoG4","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Sonnet 5.5 (max)","thinking":null,"upstream_model_id":"claude-sonnet-5-5_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"3be3e4d5ae97d1beafca7a96a0b7ad657325b3715a1928728c8c592421c78f2a","result":{"confidence_high":92.44379977522415,"confidence_low":85.10005987389864,"sample_count":285,"score":88.7719298245614,"score_display":"88.8","source_stated_rank":7},"run_fingerprint":"5ef53cbc3a8e1dd4fd17bb9dc78ad2360fa4ddcc67670097b7d7e59d12351a47","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"79e15295e728fe7ed0a6c00ba9661ee45bc1e6d291d6833048dcc0f3c5fd38fc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5d2b171b1efc0bca08030067adca5f723b2b159bf84c86c135570e77d4ef460d","result":{"confidence_high":42.942593,"confidence_low":21.821083,"sample_count":71,"score":31.4285714285714,"score_display":"31.4","source_stated_rank":7},"run_fingerprint":"7e2051ec890abeda47c1e8b564004c1f637bd257d92109b7f10ea840273b8d96","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7f66e2c79575b13a714b96fffc002dd1bbdfde7a133abf0dfb69cdcd4118c48c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"4a2b32ca40e0d83aaac6f98dd410e7e190e5d2d1a862c1d57a03eacb01c4d039","result":{"confidence_high":42.646861,"confidence_low":21.574718,"sample_count":71,"score":31.1428571428571,"score_display":"31.1","source_stated_rank":9},"run_fingerprint":"6e9be41135b21de43de348b34203df58729e7f3ba30dca3438083266735171c2","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"01d32717daed8b31f132d46904db5215fbcc54baf35144ac62ebc77bb3c0ca41","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"61ffa470c54bb065660128a2f9f11c4042141e405a8abdd24fe76addb6b3f7a2","result":{"confidence_high":35.717279,"confidence_low":16.036062,"sample_count":71,"score":24.571428571428598,"score_display":"24.6","source_stated_rank":41},"run_fingerprint":"9bce68b809bdba0a9290e24a42d17eecfe62fc9165d0faa672e474e425eada78","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c5cd2312a8fd69993f985cc79bfa9c02a3a5055e842ea80d8ae0195322def600","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"166644484abcb6ca5d8b1921927b65fd70e0460aa21c9b7f5614e58f1b93abec","result":{"confidence_high":27.209102,"confidence_low":9.907613,"sample_count":71,"score":16.8571428571429,"score_display":"16.9","source_stated_rank":70},"run_fingerprint":"e45a9b432882501aaa895f6926b31dba7cdaf0d2d74b961e6ce4c67be8674f7d","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_41b89fb2ca8e99b100b47ec0","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f1d4c82f71f72015201185d6c401f7c7fc56352f517a0ecde4e9e8085e13db23","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"dba6279c96dde96122b5be9d31a4b5c6f2879480662eddfae226a04d7f88b940","result":{"confidence_high":20.8836,"confidence_low":5.933266,"sample_count":71,"score":11.4285714285714,"score_display":"11.4","source_stated_rank":88},"run_fingerprint":"730b527449806c6867bb8b40bf41506c3fa92f68b356fe4ef593129429491234","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_76ec00376e53620e063e0f5a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4a6915522e6b0d9b5dd15e17392f511eb1369c6e392b3697384912afb1ac5607","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6971c9ccca692084616ec27794299a35637fae668727d8a0050e8400f6642e6b","result":{"confidence_high":43.027358,"confidence_low":21.762137,"sample_count":70,"score":31.428571,"score_display":"31.4","source_stated_rank":null},"run_fingerprint":"722be61287fb972fb61268a25b0cc1d90ac5b9b31bbaa20f11659f550729cf71","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9c68142453e6df2393010e3964ed7e3fa2a577c47e68a3f0d8c45919b6a508af","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-xhigh","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"ba99566ffe7f68d2fc56e2dfd0e966afde9cf3bea66439a5f3561e47429cf3d1","result":{"confidence_high":42.731678,"confidence_low":21.516117,"sample_count":70,"score":31.142857,"score_display":"31.1","source_stated_rank":null},"run_fingerprint":"3565145cc219815852bbae478f573b5b6121db8165c2c09ef3584c1453f4b12d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9eef06c2a990a1b1cc89270a0c7333cafc0450adaa027879611a9f0c77beaa77","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-high","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"114529b310377de921c0d325f67fa781c513be2045951cb51a35499251b0a18d","result":{"confidence_high":35.802545,"confidence_low":15.98615,"sample_count":70,"score":24.571429,"score_display":"24.6","source_stated_rank":null},"run_fingerprint":"2bd670b5e980d9036d5e9447f3f3afb3ecae5936df776baf321561c4c90bfa51","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"12dfef8ab274447c176c48cc4127a9fc45ac405b6cd01c5ec18a4668106adb90","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-medium","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"366851319c8da4f34ff9e9bebeb9ad37396f0102cd87a33469b5c789256cc250","result":{"confidence_high":27.2928,"confidence_low":9.869992,"sample_count":70,"score":16.857143,"score_display":"16.9","source_stated_rank":null},"run_fingerprint":"fa4c741477e801f58a88a02a3e1494ee9958d6e05788d06b52205d851cb65bbd","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_41b89fb2ca8e99b100b47ec0","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6ffe38fb9eea5de7432b9b79cb8d79a8dc17ad7bc57955846cdc401a03fa28ca","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-low","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"16c9a66b87743f1ca517c798790d2531201f1d8759f130c0ed007b42e177a688","result":{"confidence_high":20.964431,"confidence_low":5.906058,"sample_count":70,"score":11.428571,"score_display":"11.4","source_stated_rank":null},"run_fingerprint":"360173aba586b487add45f248d5121612b43c5f5f923515163669ea0c294211a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_76ec00376e53620e063e0f5a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cd0ed0a0cb7b19787421a0acb3b7844aaee617795500c2961cf4c7c13399d4d7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"9.67","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"steps_per_task":"170","tokens_per_task":"271920","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5.5 Max","source_effort":"Max","source_model_version":"claude-sonnet-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"3eca45ba5bf4097b74bfd1790b940c2f396834c61a84dd4589466946e5a3997e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.50000000000001,"score_display":"55.5","source_stated_rank":4},"run_fingerprint":"9bcdf5a3bddbd1788cd6aabf942dd3f4831a4a952ef71b7f8403675008ff5cd4","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"91b38fe35a324abdd6e56d823ce829bc90338a93f0b1d0d0a494c24fe723dcad","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"3.88","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"steps_per_task":"78","tokens_per_task":"100158","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5.5 Extra High","source_effort":"Extra High","source_model_version":"claude-sonnet-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"cb52b1719b7be9518ed541a3a86dc6b068091692378f11b7f5c52c512339299d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.1,"score_display":"53.1","source_stated_rank":5},"run_fingerprint":"903506be96e0003b0a25a98bf552178bc20839dd44f0bbeb99aa41282cd21282","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d7d5a10dcb06ce3c81962f50268315286ad18bc91adbfbd81ffdf7137fbcfb18","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.67","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"steps_per_task":"41","tokens_per_task":"37391","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5.5 High","source_effort":"High","source_model_version":"claude-sonnet-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"0dbc706f624dc08ead963925d1d922f1fa5a77c347b64c93d91a8b61499bfee1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.8,"score_display":"47.8","source_stated_rank":10},"run_fingerprint":"6547bf874d257db97b48608a8d6469ee793399b77456424c9af145474fbe6e08","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d0d700970ee301ce8e1e2feea26a8d2507655998d9e8203ec39a92bbc4cf3b52","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.7","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"steps_per_task":"22","tokens_per_task":"16036","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5.5 Medium","source_effort":"Medium","source_model_version":"claude-sonnet-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"b593849af0801fa60671ccf550face9abd6d9d3d5f53a75009a50b255e018662","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.2,"score_display":"39.2","source_stated_rank":28},"run_fingerprint":"53ceb957971182d36e5a89ca70854113311eb58343f26961dd545b14bfc940fc","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_41b89fb2ca8e99b100b47ec0","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"50bcd1d26f9b7244a5274256cbe61ba9dcea3d2974921dcbe91db6bb7182db7c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.5","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"steps_per_task":"18","tokens_per_task":"11668","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5.5 Low","source_effort":"Low","source_model_version":"claude-sonnet-5-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"39f25fd67e9fdca9a04a30e95f7e75faa3570cc5a49685304f91659475cb769d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.8,"score_display":"35.8","source_stated_rank":34},"run_fingerprint":"f131dd5ca7307038b39f2f47e52a0f770902d80a4a49512e3b546691eb8e4090","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_76ec00376e53620e063e0f5a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5c43b2cbd76b0756276b501bdd99e30bc9208208d208ed59b292e8b5b12ee94b","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5","source_effort":"xhigh","source_model_version":"claude-sonnet-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"3d7c5cfb1c3a83f418c179444db1a506184963910971b39d21696da7073e9e94","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":52.09,"score_display":"52.1","source_stated_rank":5},"run_fingerprint":"cb4b102d65bce83c5ec47b14bc31c05a8c602cc7e2a48e66bedf4578d4b1091d","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"75eaf4ee127ab8454e92b422d703fe682ace28ffa9851133282ecdd4a0b3d675","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":7.524252,"date_is_proxy":true,"latency_seconds":2751.084,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.737},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":null,"source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"e7190d2ddcc0e4a023cbabc386a9eb92cd30bdac8cea9bb6d7df696b3aac7633","result":{"confidence_high":90.38051999999999,"confidence_low":75.73148,"sample_count":null,"score":83.056,"score_display":"83.1","source_stated_rank":9},"run_fingerprint":"58d44ef8c432ce47f384e33244cb1fbfb4550ab1c609baed2f2b2f9f48611b44","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"39fe9685c91a82fddecdedd9426e32828945c899047389b246b115df9076bb1d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1076ba2cc76ac0193ce2b9eed0f7c92c04a3c7b7f948bc8f907dab6bfc6a4f03","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.9953703703704,"score_display":"61.0","source_stated_rank":4},"run_fingerprint":"0e76ba09cc99825158d3a7f45495f9a98be4449e32ecc1b9d9db3c741099187e","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"34e8d96a288d1d430379ef0a536ed6c5a2025f6a7aff07c5a42df53852b41b9f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"495cf5e465136f14bdfd98855d4c77f1b0be95b1a53167250d2074b310950754","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.2916666666667,"score_display":"57.3","source_stated_rank":22},"run_fingerprint":"166307ba3042373f201c4a615d414de9b5cc6836320e145c14d3f028efe7f608","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d960f4a150a40fc154f806f70b269978c21c673e436ab124517b87f05afbb053","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"4130b1e4de0ffe48ddd6b79b8045ecb7294ef4b071d4c2003967843bf0a8b15d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.703703703703695,"score_display":"53.7","source_stated_rank":58},"run_fingerprint":"4f9abc6c4c92bbb9ad0dae48ca3be3ebb52b1afec02a2adb83f6658154ee3c27","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"953182f99ba6adfbc38c26e920fa2f7f1f82de7b337a2554e20e9eb602c2c9d0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"c343cb6194a07e566559cf94983c3bf27c419b3f786da7b05894b6c09acb7b06","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.8935185185185,"score_display":"52.9","source_stated_rank":69},"run_fingerprint":"d6eac842f393a1d63f92560c3d99dc0c3ce41aeb9c99a2a0d4ea1ab334fae8db","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_41b89fb2ca8e99b100b47ec0","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1f349891431cffa887fe743ab8479fa3e294fba1651bde005fca72727b3fbce3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-sonnet-5-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"17929d6255197811b4b06a5963c2e7326f6ebdfbfb68c72ae0bc69096dac8e35","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.0740740740741,"score_display":"49.1","source_stated_rank":98},"run_fingerprint":"e448f86beb8559be948712377d7f4f5907d629a4d2d8e27dd6305d0d36b51609","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_76ec00376e53620e063e0f5a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eb65c44b8dba089ccdcd60dee6fa16343007abb5e49e2494b3fd2c869120e55a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"d66b0632d35c6fed6a07ef9fec0390b84f6f85408642e0df7db99f1951905abc","result":{"confidence_high":66.448651,"confidence_low":55.252618,"sample_count":288,"score":60.99537,"score_display":"61.0","source_stated_rank":null},"run_fingerprint":"bc102096c774dcc108accd51cd32c0c4e9fc6e51a9620bcab5b886c1891cfeb9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8f62ddca0bdd931d9cc81f9dc3346ebd84ad78174e619224e4c8d687958c1c70","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-xhigh","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"7ff37655dfe850c51caa951cacbacab7543235d86e67a9b0410fae347bab6e8b","result":{"confidence_high":62.87174,"confidence_low":51.519629,"sample_count":288,"score":57.291667,"score_display":"57.3","source_stated_rank":null},"run_fingerprint":"b87ee0ecd3507718d75b9b96b74f697928bdf61b41ac4f477680b94c0418fb7b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"496a2698ef440b24c5cd97afe581728cfcd874b7065b5bf7c098584c409e45dd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-high","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"65fcf9970925056037ec1100121097f35ce3ebbfb586e44d4140348d3e8fc510","result":{"confidence_high":59.375971,"confidence_low":47.933931,"sample_count":288,"score":53.703704,"score_display":"53.7","source_stated_rank":null},"run_fingerprint":"cd5cbd909caf0f6c393fb51b107c8607863ceca92027fe8323e058186b1d954d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"31701581cfab0ecc18aeb1f730a8c21ca38f52867db1635a55a78fad1ada192a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-medium","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b69135d16b0f51d6c924ebc98b4beec4478b7902496081a24ea807294ba7857f","result":{"confidence_high":58.582516,"confidence_low":47.128346,"sample_count":288,"score":52.893519,"score_display":"52.9","source_stated_rank":null},"run_fingerprint":"f540731718f83db60044c355aeb50ff909055787123e92ceab45003d4398a9ea","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_41b89fb2ca8e99b100b47ec0","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aa2348dfa5c8521129a5f59cffbcc93db7bbec77550c4add83d0ceaf78ed4891","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-sonnet-5-5-low","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"8e554399d8c533804ae1797bd3e53d1add173be01638fb4c6165580af7746886","result":{"confidence_high":54.821864,"confidence_low":43.350661,"sample_count":288,"score":49.074074,"score_display":"49.1","source_stated_rank":null},"run_fingerprint":"e3a9c0be4f890ad025e461c0b08608a3f7f20f4921be4656fefd0d1e1289140a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_76ec00376e53620e063e0f5a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1b173a4965869bd9fa105955c0ce21a92ec76da601b6bdf2bf96f22bcaa4af98","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.110921","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-sonnet-5.5-high","source_effort":null,"source_model_version":"claude-sonnet-5-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"720c31a81a42023e24e49c35e2b042b618ac030f394a64312e7e7e475578567d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1819.12,"score_display":"1819.12","source_stated_rank":10},"run_fingerprint":"2ae943bcdf98e354e52cf146d79ca8d446c548fa0510636cd02a720804de374d","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4ffbd13bc836318de96630f70edcba4e8f1c7677725d0c3e4c597e191b5d99e0","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":31.250578,"date_is_proxy":true,"latency_seconds":3672.106,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.26},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5-5"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"6d59aa19777f10a4f48acfb97b2627c328019ed8bea187fdbe715cb39f965d31","result":{"confidence_high":94.8616,"confidence_low":89.9224,"sample_count":null,"score":92.392,"score_display":"92.4","source_stated_rank":1},"run_fingerprint":"555ca3cb9f765e00ba4380d8073331caa7105732d6b9c323049d4531a01bc7ea","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b076d24f7aaed0c3b5d129250b38132350fe16fc616f3aecf9d5d1ad8a471f61","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-5-5_xhigh","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"2a5cc5f237a0c58d9951cc5192dced37ec0ea5f0453a512d9d8d966b3760ea55","result":{"confidence_high":1804.45,"confidence_low":1768.08,"sample_count":1531,"score":1786.26,"score_display":"1786.26","source_stated_rank":3},"run_fingerprint":"e3d12503e9380ed3b7b5e260bc587a8fda65cf5847ea3bc94f805b88e73bf99b","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a420b674050690db4c65aca6dc624bc5d92350145d1f9f4cce67ac0342c4845a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-28","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-5-5_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"06f2644e8a1b4eb568b7076ecbb68fd7f00eb761d68be0ec96e1418e830b289b","result":{"confidence_high":1717.81,"confidence_low":1679.82,"sample_count":1205,"score":1698.82,"score_display":"1698.82","source_stated_rank":6},"run_fingerprint":"61cd0a489035ee42090c4eb74e1d1dc247953304345325c6fb4268648e754f17","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a9d3ac47c0166146278d2d12b06a7d4b71d0a794f5f640172eb9acbcdee90f8f","metric_details":{"license":"Proprietary","variance":86.0709910606134},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-sonnet-5.5-xhigh","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b29ecdeed20468c5b177492a905b8a60c9c54201b8c8f5db8a22768bc5ce7364","result":{"confidence_high":1804.4462198241213,"confidence_low":1768.07930256045,"sample_count":1531,"score":1786.2627611922856,"score_display":"1786","source_stated_rank":3},"run_fingerprint":"dc46433a43e1c3114f8b2d8537acb9131894826377482c2bbd5eeeaa6c24a567","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"29f166dbf6e472f4b31988adf3b6ef2178b15583c3f5f3cd65fc656ef0a318bd","metric_details":{"license":"Proprietary","variance":46.52458913889888},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-sonnet-5.5-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"22f49a5ee98366072f3d486a515d7aa0f53a164aca462d4ac68616543b9572f0","result":{"confidence_high":1728.5881567657036,"confidence_low":1701.8507452734298,"sample_count":2527,"score":1715.2194510195666,"score_display":"1715","source_stated_rank":6},"run_fingerprint":"860b2a0901dcd80798987e037f2247f39a2049450e3436bc8e38d598912d0c63","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5bf6cc122ed1c06299bdcc8736b867c9af7466d0091cf3428ef03fd444739de6","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.625494,"date_is_proxy":true,"latency_seconds":465.828,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.716},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5-5"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"6270029c3472ef375e87b8168950a6918e2b017ed59aed288d7cd41d182cf557","result":{"confidence_high":86.50936,"confidence_low":79.78264,"sample_count":null,"score":83.146,"score_display":"83.1","source_stated_rank":6},"run_fingerprint":"8011a4f3652abb920da00d62c1efe80b6cf89de69cf10582258c72a24a4b4ea3","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9034e25605a2dbb12c9181ecda8b1f27404a6354e817c46b54da2ea14b6eef04","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.8370000000000001","mean_standard_error":"3.6","model_release_date":"2026-09-28","notes":null,"standard_error_points":470.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5.5 [claude-sonnet-5-5]","source_effort":null,"source_model_version":"claude-sonnet-5-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"6d8d940d8924eb3b338002090ccc769cef3dda2e25318bec9cacbd10cbddf3c5","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":75.5,"score_display":"75.5","source_stated_rank":1},"run_fingerprint":"8d5e6dd69653e59ba9f9f4f2205442fff0e25f5d906ee827b60bb08b901f4fb5","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"00600b66492bc9238a89dbeb98cfcbafcb902cee7878204e538c3191476d618e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.594","mean_standard_error":"4.9","model_release_date":"2026-09-28","notes":null,"standard_error_points":560.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5.5 (Medium)","source_effort":null,"source_model_version":"claude-sonnet-5-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"564b410a025c196b066ef51fcdfe961b4078525b5df6e0d9835afd2df6527049","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":44.6,"score_display":"44.6","source_stated_rank":29},"run_fingerprint":"5d1121deb7c4d9c69710680c95da23afb6c278ced6cd20b00745780f6230f5f3","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_41b89fb2ca8e99b100b47ec0","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6f26829b9c66e43365be4fe8e177e4936d91bad05f6fdd77484fe91a60b729db","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":5219},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Claude Sonnet 5.5 (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5d958307d3490952e796cd85f0a16e304b0980220df37d83a1a8ba2f670879c6","result":{"confidence_high":0.15612728323656616,"confidence_low":0.09428887464329698,"sample_count":483211,"score":0.12520807893993158,"score_display":"0.1252","source_stated_rank":3},"run_fingerprint":"acc63d3a1dd9f4815f0979c5fbbdabf6a44e920c1a8a98f42af16cffe393a293","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7ab197602cc07bf04a07abf5988dddce21b6635901dca568f6cd5290e173b7b8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-5","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"b0acdfb10f57b13ba475a68904bb6e130c1685d7343d871726603ed57d9e0248","result":{"confidence_high":1864.07,"confidence_low":1816.1,"sample_count":220,"score":1840.08,"score_display":"1840","source_stated_rank":null},"run_fingerprint":"d55eb71f7b75a5d98b8275081b2646d4802674ba3ca0d554ae50317e7cc31376","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_fa49f2bcbf15eab57b062ee7","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"38ee4fb9add39d591907d7dab5fde3ab882f058d3c6f2a2c3d4a99be29194cdd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-5-xhigh","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"e987386e5a5f0ca6b05347c46433a42948a2c3c8a1f80a2d0ca72a43131f2ef6","result":{"confidence_high":1753.94,"confidence_low":1702.3,"sample_count":220,"score":1728.12,"score_display":"1728","source_stated_rank":null},"run_fingerprint":"03e71b2cb24bd3850b415a61d58c2feaf61cbec4db5eb9f9e20661a288db5fe5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c047341bcdf46221302633c9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2f4e382e70c6cc99d646a816c0d61f44f5edb446278774c74c93d27c81adc9da","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-5-high","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"7cff75c17d984e48c17bb671d9bcab11a0f91c7baa11a9d7d98574113bd74242","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1551.77,"score_display":"1552","source_stated_rank":null},"run_fingerprint":"b64ed0252aa9f229e5730f593b91045603d9a42c58f76059bf4d9bbc2842bd80","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_58c2b77aab1a8f5d917ece79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"66301c4fa232a2cc56738462d3e40b80b00eab2c4d1365d2a3e5a305ec305997","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-5-medium","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"f72ff84b37c0c4b5f368fa14761c63186f3362afd77b2462f5fed3ec9a8616c0","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1323.6,"score_display":"1324","source_stated_rank":null},"run_fingerprint":"0d5611807e0815d98b8aa1a4df6956568e74280aae0f0b93f41c91c00bcd1f1e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_41b89fb2ca8e99b100b47ec0","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"39abe782413a79e7826b5968bc8c3e9f75d7c46c20c204754c60c8c2a5f2f6ad","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-28","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5-5","name":"Claude Sonnet 5.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-5-low","upstream_model_label":"Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"2d2fcc37b2bca38f967b695d02f5403733fbfd1d988caebd6a66ee5c318e6ac1","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1178.6,"score_display":"1179","source_stated_rank":null},"run_fingerprint":"ce2a3a5c04b8d7128fa3b28500614513130245dbd6fef30852b0a6bb7487542b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_76ec00376e53620e063e0f5a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aa2375e2d6826668f984b81b82decb5c4a47dd6bc0f3635a1724944a0380f471","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-09-01"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Fable 5.1; model release: 2026-09-01","source_accessibility":"API access","source_model_name":"Claude Fable 5.1","source_model_release_date":"2026-09-01","source_model_versions":["claude-fable-5-1_high","claude-fable-5-1_max","claude-fable-5-1_medium","claude-fable-5-1_unknown","claude-fable-5-1_xhigh"],"source_reasoning_efforts":["medium","high","xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a41ac9990e79b9aa745f24a3891246c96997beaebb5d81bd6cb79e1bb92cd1da","result":{"confidence_high":169.07,"confidence_low":161.8,"sample_count":null,"score":164.82,"score_display":"164.82","source_stated_rank":4},"run_fingerprint":"6f9e5edd1a2526b614c29dce9d32a226c6886e4637dad7523a7d947d46d14858","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2d5b4becde2ddac8a06f851c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_id":"epoch-eci","source_label":"reported","thinking_strategy":"adaptive"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c0f1ea8d80a6ea56ac63204ff18c692605c40c071abb699c8a9d0cf54fbce900","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"8daf214a88644afb0ff9ba38262e5ec4b497ebb93f3ab6187c2b34636459c5d6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.354926,"score_display":"53.4","source_stated_rank":null},"run_fingerprint":"65d95d299375c25538123291b0f0994f425f98e83e028cce0dada98aa8fb575a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b380bcfc6eaab9c1511a3fb16d9e2c3af46d5c0f9129cfb3ebba329024260fe1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-xhigh","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"e837bb1ae8c8d1a6cb7ef9c907d1a694e83102d0e739c578f8cca21178c83a55","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.203259,"score_display":"53.2","source_stated_rank":null},"run_fingerprint":"c83d9eb82130eab2320b735b5b3b52c5db62a9e37daddbbec0d85780a2fc68f8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9c59ff15022616da6f613e676b51a525587e21537459968effaa75ff6b9d3bc4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-high","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"952d93935e0ed39031c51d8abddf8b846ba1331317bb3a7fe562fc5239983d35","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.151571,"score_display":"51.2","source_stated_rank":null},"run_fingerprint":"c50c10b93b2d06ddf08572fb8e3cec3e6b4388f65dcc6b0aa549bd76e04f236d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"69a2bdb1718a152895c2acaa11d9d1f75bd81e0f18fb5107f355a1b02a1aad7d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-medium","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"a873ecfe8b119be7ed0c2970525650495164dc282b83606793cb97e0702f7fce","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.920616,"score_display":"48.9","source_stated_rank":null},"run_fingerprint":"aa5e51f587cc6ee7e38d1c67b055ab331d29090fc4777d50d3f54b8c154f9c48","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"35b6da510de07e2620edfd45487190705ebb0d6adfcc7474f28184177133e038","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-low","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"b37240fcb6cc53aecd280a8ff1fa6105e20ac0fcbb45ebe12782cea523419dd1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.816342,"score_display":"46.8","source_stated_rank":null},"run_fingerprint":"47f0e1c568b212528ddd511d33871519b46c9445bec17c8b18649ad158a6d179","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"383bf0e99a28d7cfe693ad4c02a323c82c4e319161bafcb2d8648ae2309fde29","metric_details":{"license":"Proprietary","variance":10.678705194673146},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1507.5404765623914,"confidence_low":1494.7308229739965,"sample_count":11800,"score":1501.1356497681938,"score_display":"1501","source_stated_rank":6},"run_fingerprint":"1aa354769f76a8298d9c1f8b5c37d79da071ac0d836eab0cbfa1b99547dd5736","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T18:28:14Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3d7f0da62f34d90bb4237a713cec6186b2d7c4f4a596555fcd9e112a0f633a1d","metric_details":{"category_scores":{"Agentic Coding":66.06066666666668,"Coding":86.375,"Data Analysis":80.27566666666667,"IF":72.98775,"Language":89.50099999999999,"Mathematics":97.00675,"Reasoning":91.69225}},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.41415476190477,"score_display":"83.41","source_stated_rank":1},"run_fingerprint":"c828448511815dff3bfd823a777b7642848ce6dfc9563cc15bc3036466111fcf","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_4e332c678b5c73c177f69177","split_or_window":"anthropic-launch-no-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"4a9acbda522dd15e3de0b5da31b3936e61aeba6d36e2ad7b8972f766fd76a66b","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","cost_per_task_usd":2.2327,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); max"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; max effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"max","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"50bdf8f6a28829e0e025199ddea4bead143976a0492e85f99029bd8d7a3cd06a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.92,"score_display":"60.92","source_stated_rank":null},"run_fingerprint":"612dc4e5dcc3fe291b2c0d67340e8019ac8e536cfb8f3d0be45b5ab7dfcc8a03","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); max","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_4e332c678b5c73c177f69177","split_or_window":"anthropic-launch-no-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"d909d30290474058739072be6fc0656848d38e070f7194b9933b01461cd6c074","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","cost_per_task_usd":1.5282,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); xhigh"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; xhigh effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"xhigh","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"873d8eed0ae8398a29d14b042d722a231dc929b51cbabd4e73ddbb890c525342","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.36,"score_display":"60.36","source_stated_rank":null},"run_fingerprint":"3db5c7cf06df4ba6d52f26e4f4862aeef0e9bc2e2763db72165ff722cc6d3006","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); xhigh","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_4e332c678b5c73c177f69177","split_or_window":"anthropic-launch-no-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"a41a1885e8bbd54bc722a76c3a63aeb9be98d829f93a2bda12584ab6e5ee8a09","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","cost_per_task_usd":0.7517,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); high"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; high effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"high","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f0886b89a5a43982d3f759d6bc91a5fe2544be662057d8bac556cfd7efb24965","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.96,"score_display":"57.96","source_stated_rank":null},"run_fingerprint":"a02deb40669ce9251b253706f6f265dd08699888202df3ca432e96ebad026432","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); high","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_4e332c678b5c73c177f69177","split_or_window":"anthropic-launch-no-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"ab801f3844688ea990cb8a5a873bcd99ae7859162b0b3aa0150b9df5b55563b1","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","cost_per_task_usd":0.4632,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); medium"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; medium effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"medium","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d4da4f55c9c42ca97757af186c5034f0b038e08b3d6b7eff14971241abdda04a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.92,"score_display":"55.92","source_stated_rank":null},"run_fingerprint":"f097704daf6a6ad70e992c30b3a33ebe71766e133d6048891f346ca2885bb208","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); medium","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_4e332c678b5c73c177f69177","split_or_window":"anthropic-launch-no-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"4e1167f94faf97413f5829f131ddad0567a42e2a783d4e5254de0a65afa19740","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","cost_per_task_usd":0.3002,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); low"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; low effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"low","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2672369739db271c4cefdf42de23b03f31c3ad497f1a91656303548f87a0ff00","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.16,"score_display":"53.16","source_stated_rank":null},"run_fingerprint":"055fad8bed6c37f798cfa315cb8257421c8c6a233def8d5f0bddd53546efd8e8","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (no tools); low","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"evaluation_at","evaluated_at":"2026-09-01","first_observed_at":"2026-09-01T21:01:36Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6e6df124449e32f8aeea0065a7e4115fd4fa0db5378f51fa915660448d7a7ffa","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/anthropic-claude-fable-5-1","source_model":"Claude Fable 5.1"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1dbc2f047b37cd251d2e6c6728a48f2776bda439be12d7e3cf2d4a3b93057bb8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.9,"score_display":"60.9","source_stated_rank":1},"run_fingerprint":"8e87ef549585b986ee515e44f9ff5b97da8e6b4d344af8afdf6c3dbef80670b1","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f0e0fcd9ef2bc38ab08e6b86","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_id":"evals-report-hle","source_label":"reported","thinking_strategy":"adaptive"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_f28de61fd7751e25719eaae0","split_or_window":"anthropic-launch-with-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"45f3b3df527d4beea5ff4ee0fed6e5529bbcad055aa43a67581408f73a42bc7a","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","cost_per_task_usd":3.1957,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); max"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; max effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"max","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"provider tool suite (details not reported)"},"run_count":null,"scaffold":null,"tools_allowed":"provider tool suite (details not reported)"},"protocol_fingerprint":"aa935ae72a6b304821533bc9c525d7f3391240376db09d85e41013309e017d1c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.0,"score_display":"65.0","source_stated_rank":null},"run_fingerprint":"04d728c810307d0317a48f7cc4a3ec1728beae764442bc842a2808a39f9c6888","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); max","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_f28de61fd7751e25719eaae0","split_or_window":"anthropic-launch-with-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"6f0f6601a2ab958501f7a6542b8e8eda6f879ba00ae1fef2d9f4d1b77351d9a4","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","cost_per_task_usd":2.2783,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); xhigh"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; xhigh effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"xhigh","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"provider tool suite (details not reported)"},"run_count":null,"scaffold":null,"tools_allowed":"provider tool suite (details not reported)"},"protocol_fingerprint":"7e5b0828e17e4c98954b05d49ec4ebf9bf0274732571e9f53786ca7d9597ec29","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.08,"score_display":"65.08","source_stated_rank":null},"run_fingerprint":"d703568cf0f3598ab872a1e5fb0f09b1e523c3b7d1fc98c9e43fc405f9a09dea","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); xhigh","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_f28de61fd7751e25719eaae0","split_or_window":"anthropic-launch-with-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"7d5d0dfd18634acc427c16254c0932c3a8f1873946e4a16b8bec9a27b38737f0","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","cost_per_task_usd":1.0498,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); high"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; high effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"high","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"provider tool suite (details not reported)"},"run_count":null,"scaffold":null,"tools_allowed":"provider tool suite (details not reported)"},"protocol_fingerprint":"a826e81fc78c24c086a4b056c2b86fbd5befee5efd721a6141c73d774dcfcc80","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.76,"score_display":"64.76","source_stated_rank":null},"run_fingerprint":"32920fc551ba05474175d799f309502601d401db9d5627acc963c19fb88e59ca","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); high","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_f28de61fd7751e25719eaae0","split_or_window":"anthropic-launch-with-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"5896f80adc912abc02095a40d6ffe8c32735e560eed1fa32c1ee1277c93cb2dc","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","cost_per_task_usd":0.6719,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); medium"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; medium effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"medium","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"provider tool suite (details not reported)"},"run_count":null,"scaffold":null,"tools_allowed":"provider tool suite (details not reported)"},"protocol_fingerprint":"3dee2bee32ba68125446c4a71842d9c17a7f55971f98958b1dd5294938ed099a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.96,"score_display":"62.96","source_stated_rank":null},"run_fingerprint":"65f7824543b21f3649af3e74919a71fc3217c5bc268659ed5b31f48ba38ed549","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); medium","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_f28de61fd7751e25719eaae0","split_or_window":"anthropic-launch-with-tools","unit":"percent","version":"HLE · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"a853b823e1df06e846a01b50b81f16d93fe4c57129a3ec2c82c6fe79790819b7","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","cost_per_task_usd":0.5237,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); low"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; low effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"low","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"provider tool suite (details not reported)"},"run_count":null,"scaffold":null,"tools_allowed":"provider tool suite (details not reported)"},"protocol_fingerprint":"53fe8dfecd06df3d83f1f0327adb312666e8b42d4b0d2946ff3a78c5114f8c95","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.0,"score_display":"60.0","source_stated_rank":null},"run_fingerprint":"cdbeca40fcf4dff47aacc947b241b18a60ae3ec67507d3be1fd113e03b6d3c80","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Humanity's Last Exam; Fable 5.1 (with tools); low","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"26ee283d14be73a88ffaf6a5a4b48a9b9b9b510e5d1adf0675db40126fb7bcbe","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a3dada637aa1d14e9925744db4c63050cad680cf385d70ae08e87982824997b7","result":{"confidence_high":61.184963,"confidence_low":57.040239,"sample_count":2158,"score":59.128823,"score_display":"59.1","source_stated_rank":null},"run_fingerprint":"a2de2bdaba7bab60a3945f4abb96c6e3faf4ff64e7c3665ec22ad136e4c95ac0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dc81fec0821dc11bde28eae69d5ce121ca041d5f53a57c54cbaf053dada615df","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-xhigh","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9df67f6e263c326843e72db0bf21a8c6bc32c2a7e679e728937fc11ae4865bc9","result":{"confidence_high":60.771833,"confidence_low":56.620745,"sample_count":2158,"score":58.71177,"score_display":"58.7","source_stated_rank":null},"run_fingerprint":"5cd6ecdd4b2ae45bba513cc7d197963cf9ec3b3ec356c7ba0fbf3d5438bb01b8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f735cbef057c09aa472cd18268a86862b68a03e60892894114146ee39772e34c","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-high","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"30faecd60e545c73ece7d50cf07fe5abb4ac1d7bf6a7f7f91dd3af12baf9a786","result":{"confidence_high":58.013747,"confidence_low":53.828009,"sample_count":2158,"score":55.931418,"score_display":"55.9","source_stated_rank":null},"run_fingerprint":"19ce38145dcaf6cad1a5c5cd4f5a54db50345634f0a667782baacfb6eb892a92","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e220e3174c0ac48f70b53742a1d80c1610d4b0946796d8c225a9e6720f7aa42e","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-medium","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7837b42066e72fcbc23cf057e916e854a63728754f429cad007d9c20ce28d3f5","result":{"confidence_high":55.894704,"confidence_low":51.691421,"sample_count":2158,"score":53.799815,"score_display":"53.8","source_stated_rank":null},"run_fingerprint":"693dd2427fde9e5335506c8352bb18c6f6395d0688cdb39a7eabb62424e77673","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fc994693f5306477c2bed89d62da42005deffe7660756274193b35a55ac28902","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · hle"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-low","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0f8b89dec5aaf1f007c2e995899c187e3f780b7ab2ecae47264af4b68d97b9d8","result":{"confidence_high":50.997041,"confidence_low":46.78263,"sample_count":2158,"score":48.887859,"score_display":"48.9","source_stated_rank":null},"run_fingerprint":"d903408bb312387c4e3821d00515418cf4485e31dc01f9d2f28922616c71ed0b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-01T18:31:59.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-01T18:31:59.000Z","first_observed_at":"2026-09-01T23:42:07Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"457428e3621ce31adca8b6588f88d6f16b457493cc6cf98f248665c0f97a0a97","metric_details":{"best_score_across_scorers":"0.708","model_release_date":"2026-09-01","stderr":1.4385511563477433},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ExyL3GN8s34tizrXV3VmS3","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"37bef27d5bdce691247742e00286686d38e2dcdb9b6fb961c64286a4c91aa07f","result":{"confidence_high":73.61956026644157,"confidence_low":67.98043973355843,"sample_count":1000,"score":70.8,"score_display":"70.8","source_stated_rank":5},"run_fingerprint":"43822f7100405a34906075b16adb5925a653dc37a3183c40e721660f36342a24","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f8a5aa39331b68d97301395fad9ec9a74dff16142912dfc5af9b311a835e4c67","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.204523,"date_is_proxy":true,"latency_seconds":45.857,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.863},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","few_shot_accuracy":92.929,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5-1","zero_shot_accuracy":93.939},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"e47af76e092338e1b53e47f692190a9378814b2220def2393c2ca068b21ccb8e","result":{"confidence_high":97.08548,"confidence_low":89.78251999999999,"sample_count":396,"score":93.434,"score_display":"93.43","source_stated_rank":7},"run_fingerprint":"0bf8d28a99c9005da9bddd74abe1f746764afb1187271ffbaf257ed0ad621cce","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"85e407aafc0cac2c8c9bcfab5eb3ee744fd4a0637a417476c34f12783edc213c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · gpqa"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2f6ed39231e516060e359d291437dc87aa452f5d1a79b6459a4c8d17d550c14b","result":{"confidence_high":96.349635,"confidence_low":89.460228,"sample_count":198,"score":93.737374,"score_display":"93.7","source_stated_rank":null},"run_fingerprint":"c4acf90780cf838e3da694ea59c205e3c3f6df1fb3ffd2dd2f4d636a633ed0c6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"94289c444750370c5eefcff6b9b7a7b6179023692ba77ca2cf48ee78e7d24c81","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · gpqa"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-xhigh","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"59271ca75af245fa9d0c86b7ebed41a89b287bb1b226cf5cb2b848cf4ac71b29","result":{"confidence_high":96.123231,"confidence_low":89.092106,"sample_count":198,"score":93.434343,"score_display":"93.4","source_stated_rank":null},"run_fingerprint":"0a0acb6d40f562e7262da32a0f17303fce26f9b19345896d8926a4c9cafbdb29","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"40a0e36cc839387907832ca4fa2fe3683c767ce505b73ba5831b40cc978daeaf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · gpqa"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-high","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2b152c67ac8f5bb507a0e9eb9082c442eb32e37634513fef80516abc1bd78f47","result":{"confidence_high":93.931627,"confidence_low":85.734806,"sample_count":198,"score":90.606061,"score_display":"90.6","source_stated_rank":null},"run_fingerprint":"641f4576cfd113058bf32416d95ad7e410e81e929c18455dc041a7c553fec628","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2680dab02392b282cd12cbd9f22ac99d16cc0083e1dc5030bd364263d7f27c72","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · gpqa"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-medium","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b7309a4e6d9e9fbf88d1f3bf6b34b7b8c716a2ac8284f2df92205a6f3527fa24","result":{"confidence_high":92.299379,"confidence_low":83.403549,"sample_count":198,"score":88.585859,"score_display":"88.6","source_stated_rank":null},"run_fingerprint":"983cb5377f160889e9f57f365149512a2f02c7924a9606fc464635ca97e4f045","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ca86fcd3e38123fb9f8b1f874367a93fe980798ad16a4b0d56512a6bea516b82","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · gpqa"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-low","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7473b6c6ca8ae01ae9b9f7c96927735717dc3deb1d0163bfe8258cf6e6f917eb","result":{"confidence_high":91.884478,"confidence_low":82.827574,"sample_count":198,"score":88.080808,"score_display":"88.1","source_stated_rank":null},"run_fingerprint":"39178b5b25eae3469f478fec79c24371b9366f971bbd61afbddfa9c912576b86","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"3e40aa98bf50bad33593c8a24cc6c0c0cfefce5285e0d876b8b2690abaa61bb9","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.097724,"date_is_proxy":false,"latency_seconds":20.589,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-fable-5-1","source_locator":"Results · Overall accuracy","stderr":0.266},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"573dcfdb9c6de85172567d2cc6fea41f95848ab9725b1744c347393478cf76cd","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5-1"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c01697c258e0dc2d236a73befd111f1dbc88ef74748c19cecb1e3409c8577ebd","result":{"confidence_high":92.89636,"confidence_low":91.85364,"sample_count":null,"score":92.375,"score_display":"92.4","source_stated_rank":1},"run_fingerprint":"096432954b6ed8314a806130fb5829848c5839a81f5d519fa4807b1bd22fe9ba","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-01T18:32:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-01T18:32:00.000Z","first_observed_at":"2026-09-01T23:40:40Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ed687eff08b8549a818b4820eb7ac233542644b421050b11c7749f4a7981f9b9","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-09-01","stderr":0.0},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mRAASXsUuoJmc2oR5G89zR","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"db287715e5e4ff598049f787f8ce4dd10a0501eded5d1d5d2ab14adecb1d11c7","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"e8330269ff653e81fdfc9c6755549d05cd01b7c602268e1d368f7fa681b234c0","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-01T18:38:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-01T18:38:24.000Z","first_observed_at":"2026-09-01T23:40:48Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"db370f0c1375126c8472e466e92a380ab182a26e0c4b03b47abf535629404628","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.9017543859649123","leaderboard_private_problem_count":285,"model_release_date":"2026-09-01","public_example_count":10,"stderr":1.7662067789031757},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"g7dXwZH2tWvqLg3X5idGs2","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Fable 5.1 (max)","thinking":null,"upstream_model_id":"claude-fable-5-1_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"14bcd4f8da0ac33a2bb0638ecd047edaa58024aab82dbe53bdcdf9bea305f45d","result":{"confidence_high":93.63720388314145,"confidence_low":86.71367330984101,"sample_count":285,"score":90.17543859649123,"score_display":"90.2","source_stated_rank":4},"run_fingerprint":"ca05b85c60106fbff1766a0768a4eb5e15c5b31b5e108140aa992df75fd93b41","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4688dc81b539f292f822f5c11e9c9dbdd71db0e8b90306b62b2714f19643e473","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":4.492348833333334,"model_release_date":"2026-09-01T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5-1"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5-1","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-1-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7fc618538143bfc630a5e98873be0750b7be8cd7799576d0787347fbd031e46a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.0,"score_display":"90","source_stated_rank":14},"run_fingerprint":"2eb337e6a4e1f90ed2f37e7263f3cad8d5efba13ef80b0851c0c0f349cc70986","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5cf3e74cde38110ede8a638e0815e31da8132f0f856d2be987135b0402e6c2ea","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":3.120216916666669,"model_release_date":"2026-09-01T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5-1"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5-1","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-1-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"049c615b58f248b50da99487e0781451eddb79a5f2984a49346dcaac5ecd44d9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.0,"score_display":"90","source_stated_rank":14},"run_fingerprint":"9006f3be6cfad447d1043df5bec3e327cf72bc9342d1cdb386be7bec2a08ee61","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d7001d3be2fbb8cab3b1eeba09e092eb31dd860ad7850d2d06fdae7800cc1390","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.6673061666666666,"model_release_date":"2026-09-01T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5-1"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5-1","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-1-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ff1425c85f042c8e5586886f0bfaf703b7a6480db3d5198788f1af339d533f88","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.75,"score_display":"89","source_stated_rank":20},"run_fingerprint":"c8bca2a6c7ab921c0a4f072e0caad9afbf5aaba3186194ee1898a28814d628cf","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e433028f4f75f019b4cc91677c3f41f02b97cc04a03d09adabc1e278956ec419","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.2239411666666673,"model_release_date":"2026-09-01T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5-1"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5-1","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-1-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"046439f93e6a595edbba808488854dc72fbae8f5227e3128b028e98e53ff3e4b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.25,"score_display":"86","source_stated_rank":26},"run_fingerprint":"2811d0e20c09d62d7d889da493f357756b260df36b75c896146172e7657253e3","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bdced206c9860b133d1f7a6cecf7039d58b94e10f9574c89efa1c4818d1df1b5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.9523265833333336,"model_release_date":"2026-09-01T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5-1"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5-1","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-1-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6e818d626b1855132cfd78bf9c4ffc09628dbdc1c3f07155d00666c86f8c68b8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.33333333333333,"score_display":"78","source_stated_rank":39},"run_fingerprint":"659d22e7b9d03cddd0e3faaa36c27d5925c144216bf701da9de6e3bbd82bcc06","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:45Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cc0c85118280b1d33e99386296fb5a62f8ddbb46f6824900e81a213e5c7d687f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Max)","source_effort":null,"source_model_version":"claude-fable-5-1_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0224617c220a21d7ec8820c7a27f7bad1786d420677887784d69e9aad76b9f7f","result":{"confidence_high":92.689087,"confidence_low":86.466237,"sample_count":360,"score":90.0,"score_display":"90.0","source_stated_rank":13},"run_fingerprint":"1a6e1f9edd4d77046bce8d6761871acdb0411db1a16bbdb97eb0f6873ffc079c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:45Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4ed45b0c3ff10876e2d50dc9192b813e2ba3d68cac3d81b1e5fd0b6b70e1f2c6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (XHigh)","source_effort":null,"source_model_version":"claude-fable-5-1_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e1c7c7a98959e7c6af69e7bf2e1a5acaa32dc8ede871b58bc8ede75c8011b5d1","result":{"confidence_high":92.689087,"confidence_low":86.466237,"sample_count":360,"score":90.0,"score_display":"90.0","source_stated_rank":13},"run_fingerprint":"29815ce5b13e3b63c8f9df46dbdf6f92f38bdffb0f15e086f36e54219f296d74","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:45Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8fcb0c80268daf92ff7ba556614f82d5d0b8a6d39a3b15f14a661b8ec13dbf16","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (High)","source_effort":null,"source_model_version":"claude-fable-5-1_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"2f666d69e86cd08c22eb2b14d8c108d1c52830c4851357e39e4124f6c2cd5030","result":{"confidence_high":91.613373,"confidence_low":85.068348,"sample_count":360,"score":88.75,"score_display":"88.8","source_stated_rank":19},"run_fingerprint":"2e688a901c64b8defa999ab1b442be8d39af1f4572f23b5ffa47c1dd9ac6d6ab","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:45Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dedcbf7612271b59393c246a2838dfcb9013ce0c71d6cafb22fb940860e0307b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Medium)","source_effort":null,"source_model_version":"claude-fable-5-1_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6740ba9e8b6715d340855c99b72054476b5e25ec58c12291156a27aef73274e9","result":{"confidence_high":89.42649,"confidence_low":82.308023,"sample_count":360,"score":86.25,"score_display":"86.2","source_stated_rank":26},"run_fingerprint":"a4684562c85291adf1a8b6ab7c151d1088e9aab23e7df5e165af532b669a9b77","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:45Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"93423aee8d5f10b75446fb12e8ac42dd080382a4e729109ee75548cf0f22f21e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Low)","source_effort":null,"source_model_version":"claude-fable-5-1_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0e3b26980121330330e905ed27ee1b1677b3ee6373b7e88da0be267059d63186","result":{"confidence_high":82.277939,"confidence_low":73.790416,"sample_count":360,"score":78.33333333333333,"score_display":"78.3","source_stated_rank":41},"run_fingerprint":"6f9515f8ddbe641bed1939652de55f78b5248b1f5796e9d1e38f3d747718753c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-01T18:38:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-01T18:38:24.000Z","first_observed_at":"2026-09-01T23:40:51Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"84af28a1891660758057cf49c3f69c6a2420570c548388d077ebe6d0db5404f2","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.8780487804878049","leaderboard_private_problem_count":41,"model_release_date":"2026-09-01","public_example_count":2,"stderr":5.173952057462543},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"T56N3X4KLA3xz7tf5aCMGb","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Fable 5.1 (max)","thinking":null,"upstream_model_id":"claude-fable-5-1_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"fcd0c753c48d67aa1457fb58f84ecd176bf9f909c0168f7ca1082626781e83e9","result":{"confidence_high":97.94582408140708,"confidence_low":77.66393201615391,"sample_count":41,"score":87.8048780487805,"score_display":"87.8","source_stated_rank":8},"run_fingerprint":"00da65b1a3a9d7aa8918f1bbd58b23cb0a1839a5577518870988333fd7f84efb","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"67c05eeb566cb2fdb500a9669895e6c952b4150c383de1a136a9c47a93004bbb","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.166285,"date_is_proxy":false,"latency_seconds":37.47,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-fable-5-1","source_locator":"Results · Overall accuracy","stderr":0.7},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"809c2c69b0d06ddc9a1abf8951e544e1554477d59ad88a1ec8b6a541a7caa968","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5-1"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"76f29f9286130688bd247efec85f16ea77e431667beba6b29dbd7b1f9264935d","result":{"confidence_high":92.008,"confidence_low":89.264,"sample_count":null,"score":90.636,"score_display":"90.6","source_stated_rank":1},"run_fingerprint":"30fcf84afe35a1f3843581bc1b409029f8a886f6d5e4355b85068efd4a2f6948","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"67cfac4501d2b0525151850afb98a3ce06dbde0f5eab90366393863703019736","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8ec14660e5f8d30f21cb5f4c1cad469c66cd3ae81aff16ba7c99b663cfead21b","result":{"confidence_high":41.161596,"confidence_low":20.349497,"sample_count":71,"score":29.714285714285698,"score_display":"29.7","source_stated_rank":18},"run_fingerprint":"2c0962ef2652a2c40293219defa195be2995416887da57623411d2d3e5266f54","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4f889f88bd5d70c0a27af00a38be2f8691e8ec58c5e93b22ba11be2d213f4d82","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a64d3c54d3c0531db021ddf667a2e2802b17f6049f811264f565fdc6a11fb364","result":{"confidence_high":42.646861,"confidence_low":21.574718,"sample_count":71,"score":31.1428571428571,"score_display":"31.1","source_stated_rank":9},"run_fingerprint":"a2549609374c80944a2cd097d4abcadf21f8de7e1a8d9ec6bd2c1e184bf2b491","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2b3f974638adb5059fc2eeb8133732b18493c1cee3b403a6aec4b6f0d6442790","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"bc6e933cd173d75194f7ec7622f3e70df0198617af7759aed613322f21e792a2","result":{"confidence_high":41.757036,"confidence_low":20.838251,"sample_count":71,"score":30.2857142857143,"score_display":"30.3","source_stated_rank":14},"run_fingerprint":"84c13ccbf50431d50b3ea5af8265d07e0d0f51eb665e9c4d380ef5dedb51af87","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ca3f91d34abd30919b469d0069de99b6bebfcd955f79c82795ea814894073d75","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"47672f9d5c3238687597c91f5f70b556907f5348956dfac8b8f56e1b63b1b793","result":{"confidence_high":40.564344,"confidence_low":19.862553,"sample_count":71,"score":29.1428571428571,"score_display":"29.1","source_stated_rank":19},"run_fingerprint":"28ad9bd09f3d9b959b1c49f666f0c9c2ddd97e9c3f3d11546378794a6275e46f","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"71a74a3174936c4585f11e7c863f334c80bd26d1015bea5d8594004c5a29cdc3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"87cd8aab04147b89f83440ad634e64abc095c8c3199a5ce4980b6c3d1d14e27a","result":{"confidence_high":39.063098,"confidence_low":18.653313,"sample_count":71,"score":27.7142857142857,"score_display":"27.7","source_stated_rank":27},"run_fingerprint":"6724b2021f472c70f4eb6cc171ecca57eedfd6c95ad2759b0684bbb847da4de1","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"be1e79e94dba97ce2d6a6b866d53a1ec589b0029d8d457ba270b03c61883c4d7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"feb5c7d05db12b4c05d05245d2cebdcb4a854e6defaf8365f01a5ab191132246","result":{"confidence_high":41.246632,"confidence_low":20.292663,"sample_count":70,"score":29.714286,"score_display":"29.7","source_stated_rank":null},"run_fingerprint":"ba22681ecf4f64d65c64febc7c15c6e8d59ceea0f0dec451c0b5b6972e0b9c0a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7612c27bbbd0e1d5970647d83246810443ea6d30e35e6c6873f0387e8d60c94b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-xhigh","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7032d201e15b3d0ed0141f4439b75a844a064db2d4fe9f8b52d540ed9d3b998f","result":{"confidence_high":42.731678,"confidence_low":21.516117,"sample_count":70,"score":31.142857,"score_display":"31.1","source_stated_rank":null},"run_fingerprint":"1a8d69e0653b4c5498e967d0314082f2af6f222a085d4d33beb0e6936f0af5f3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d16bb75fbdb45c3e274f99f88f3eed31acc6e336f9e9acf5d51201dec62c2f2b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-high","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"49d8388ba9399f4e6de4eecb942bac0d0bfc3b49b273014d5a61eddab5e18273","result":{"confidence_high":41.841992,"confidence_low":20.780702,"sample_count":70,"score":30.285714,"score_display":"30.3","source_stated_rank":null},"run_fingerprint":"3694e778068b0c9aef5cde7363824464627aa5865191c5d7ecc0edd66a9419e8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7a47b8ae5675186314a5a2f5bf6101f6d493ab0ad23de009aa8df298cf946893","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-medium","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7f24116de332391cec41c88641c4bdc685c35b9c06921de013e47c278720521f","result":{"confidence_high":40.64945,"confidence_low":19.806444,"sample_count":70,"score":29.142857,"score_display":"29.1","source_stated_rank":null},"run_fingerprint":"bca7bd5500ad65897795abe4c159acdcb01e414217f8813aa277db5ab0ddf48b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c77bb7b2d1783e37631ddbb70311a22f9c6c031a4645304bffa9ac4164c306ab","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · critpt"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-low","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"3ef986bc6cbf880a9a858fb132174694d8c8a595cc9794927d75289c389f472e","result":{"confidence_high":39.148332,"confidence_low":18.599063,"sample_count":70,"score":27.714286,"score_display":"27.7","source_stated_rank":null},"run_fingerprint":"c047793a9d24a8f9a32848ea9bb9d96cdb7293b934b12a94f5b8d5a59a1f0e38","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-01T18:32:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-01T18:32:00.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f16db4c88524a2fbcfc8e258b2cc69d92a674765fd54be00ccf2332e3ae5cd46","metric_details":{"best_score_across_scorers":"0.47","model_release_date":"2026-09-01","stderr":5.016135580465917},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"EEGf5SDiF8jXB6NX7eR6Sa","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"de6b150827b7cb06cd09eeed4b6f0f273f1612996f554c325b5b0bbede6e4a44","result":{"confidence_high":56.831625737713196,"confidence_low":37.168374262286804,"sample_count":100,"score":47.0,"score_display":"47.0","source_stated_rank":14},"run_fingerprint":"8dbaa68bca116458fb4ebace7ab70aa696542713eece2fdd23cdbe81e9de2b55","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:12Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"df4613fd5ec34b749b5cfa13a3096529e6c96ac8e6fa0bedc293d4adc55d07cb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Max)","source_effort":null,"source_model_version":"claude-fable-5-1_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"698663a4078ad821dc00c6db456b1c588d0dd1db5c52ee8a1b5aeb8dee0a4397","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"43f0d59fad0161823b7df085d20150332956ae16eb825f42b16f629110813209","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:12Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c60aa804e636ad8abd1afadd35f5c46484ee62ddc2ee5a9a29cbfafbefe52037","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (XHigh)","source_effort":null,"source_model_version":"claude-fable-5-1_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"8d5d7b0d117da4fc74383fccd63b4d481810f3317b09bff988a2c0c587e58f1c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":96.5,"score_display":"96.5","source_stated_rank":21},"run_fingerprint":"689ac40671f1b4b3f7fce1c0546bb713e1a885b03b359f3a76374bd16f576f18","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:12Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"65f1d2180734c79f8fe74c570130a315147cf24e6cc0aa5f5cf854ce20389394","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (High)","source_effort":null,"source_model_version":"claude-fable-5-1_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"580663321d9b2e41bc9cedcc2dc11108003dcbbc402976c97d11cfb4bb729562","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":96.0,"score_display":"96.0","source_stated_rank":27},"run_fingerprint":"b5b64f6f230d29b9b006c49b1f60c96815afd89b60731bb983ff52b2f8f11715","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:12Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b786cc36c30e78c88951fdd549c37f6a7ce584906cec2641de6b4036fa2a9d3b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Medium)","source_effort":null,"source_model_version":"claude-fable-5-1_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6ac009680c07ad7e43fd59b6ec69c0f681e68bbedd316a2729c1a7da18117d1b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":94.5,"score_display":"94.5","source_stated_rank":37},"run_fingerprint":"80ae36dcf625eb3c243477d235a9cb718e33b57f4661c2c52687c20b98567cbc","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:12Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:12Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"32551b517e61c60336e45bca593471c95f997744f9727f39e0c0c7682e4a8b1a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Low)","source_effort":null,"source_model_version":"claude-fable-5-1_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"db799008e8f7dda75f5283a08308e641ef52bf65dc0aba6189c91e902e84acb7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.0,"score_display":"90.0","source_stated_rank":75},"run_fingerprint":"51b17cb473595eb0c8b46dcb35433d69620c7398ae303bd0e01ecb4d07e19815","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"20b909107aa8730f079cfc81183f50c5dfdda594c14b4ee7a2cd8782d01350cf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"17.28","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"steps_per_task":"128","tokens_per_task":"117236","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5.1 Max","source_effort":"Max","source_model_version":"claude-fable-5-1_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"648ff035bc2bb6669ac88379b3a9b625dbc4e318a33fbded080fa7224edc3373","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.800000000000004,"score_display":"51.8","source_stated_rank":7},"run_fingerprint":"5317b0a0cbb543d49249fb13b28d201e1ee4647bf0df974e5b2336109d6bca8e","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2ca27f0efd650b3429788078e5231248c9a24c1523f5e8731023e3c76f586da6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"13.01","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"steps_per_task":"101","tokens_per_task":"87294","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5.1 Extra High","source_effort":"Extra High","source_model_version":"claude-fable-5-1_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"9d0dee575c642240eaf4e2c3ecec0c88fc5471f9257699963938bb973c48ba88","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.6,"score_display":"51.6","source_stated_rank":8},"run_fingerprint":"8de0c2ea69eed3c53f85d96a56fe583d2c57a34b77430da1efa490d7c8c3e635","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0d1f0636f7d944761bc7457204a6664013bca98973883b8551d249fb544a0bed","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"9.08","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"steps_per_task":"77","tokens_per_task":"58438","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5.1 High","source_effort":"High","source_model_version":"claude-fable-5-1_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"9ff6908f2701464901d943f33c0dcbd4e6350246f54819b6c527f78ba1988cdf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.2,"score_display":"49.2","source_stated_rank":9},"run_fingerprint":"743f9f4d3986289a88ab10539e7ed1838eec75f1794054808cfd78a24e36444b","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"55a278e734d7b3388b52c6e056e333480fa5b4b1e8f0323a84eb3f3c7ce28454","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"7.05","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"steps_per_task":"63","tokens_per_task":"45411","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5.1 Medium","source_effort":"Medium","source_model_version":"claude-fable-5-1_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"4439bb26d5f4596d5f0c9976decb860bef7f1372394566958ff0bf301129e31e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.8,"score_display":"46.8","source_stated_rank":11},"run_fingerprint":"1a38f09f1b12071282419a75962050130cb4cf9ced4797f10b2942dcb78c2493","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b52ca02ee616e4f9f7e2908f695f5279862c71d517652949f6ed572eb8b4f474","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"5.44","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"steps_per_task":"51","tokens_per_task":"34795","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5.1 Low","source_effort":"Low","source_model_version":"claude-fable-5-1_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"ba3742f686360efa902b59b5c656731458a9c20d34e65350aae3a89f6df6f7da","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.1,"score_display":"45.1","source_stated_rank":15},"run_fingerprint":"b2fe85a48da22b0d01b6fab92c11c8b17ca1a19cbbffc028f6a4ba00066d6e7f","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_d30d0e06ff371d04e04dbf66","split_or_window":"3.2.0-provider-run","unit":"percent","version":"CursorBench 3.2.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"61b703a638a6530ed07b2a48ca57e61fe783bb0fc9064f697b622df522701b3b","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":9.64,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; max"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; max effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"max","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"coding-agent tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"79958d0c430fb1c249910d59b1af90148b9da6b1fba39f5d71e7ca54ec301662","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.4,"score_display":"73.4","source_stated_rank":null},"run_fingerprint":"3d1be83dd2baa28e956ac16b09ef90cfc5ea071d4de8924ca74de42991f1c627","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; max","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_d30d0e06ff371d04e04dbf66","split_or_window":"3.2.0-provider-run","unit":"percent","version":"CursorBench 3.2.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"d6f0073bf054f9e789303ea17ad606363678838bff3326d52d14ba64fd6c2210","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":6.96,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; xhigh"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; xhigh effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"xhigh","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"coding-agent tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"19d7695d435dd3aad2b129ed387944e8f8fd7e25ed8d25166610ab3d03794917","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.8,"score_display":"72.8","source_stated_rank":null},"run_fingerprint":"c775ffb006c0e53b55400aa6301320ffd81f20feed1558c38d2e772f22c00c8a","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; xhigh","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_d30d0e06ff371d04e04dbf66","split_or_window":"3.2.0-provider-run","unit":"percent","version":"CursorBench 3.2.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"67e000f0c1c73ccb51a905d350d5771e3d9dcfd5c976b105824982dabcee5527","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":4.8,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; high"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; high effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"high","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"coding-agent tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"e937840dbb75a9343408951edb7fd845264d064a7fc988b653d3b6a78087c11f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.4,"score_display":"69.4","source_stated_rank":null},"run_fingerprint":"10d203075e7d3f33455d975071f1c915f85ba1a1fa480aa595ca3389f4d9ba1f","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; high","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_d30d0e06ff371d04e04dbf66","split_or_window":"3.2.0-provider-run","unit":"percent","version":"CursorBench 3.2.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"d6b80ae3085faca184f10513b599a03584583ec46fcca4d2d84f7a2f59364600","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":3.53,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; medium"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; medium effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"medium","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"coding-agent tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"56be642f7a54f3ffdb20a920c657ba6eeca9f6658feef956aeb46a65ac64ffcc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.0,"score_display":"68.0","source_stated_rank":null},"run_fingerprint":"5aa861773e96e95c2e9d1112ec14f907c6d48618c063fa0b02728b1c94381af2","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; medium","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_d30d0e06ff371d04e04dbf66","split_or_window":"3.2.0-provider-run","unit":"percent","version":"CursorBench 3.2.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"70aac1cb58a5bf04b3c0966adbcd099c27ffbebca927a8343ae16379598c1b0a","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":2.9,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; low"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; low effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"low","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"coding-agent tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"d76f87459fb94d28112d35253cd9f28826bdb68b74c721bba1e76bb9425d8e29","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.2,"score_display":"66.2","source_stated_rank":null},"run_fingerprint":"546b5b59efa393cfe65950cd0d3081034d68a0a8e021c6f18efe13162d8b3d1f","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; CursorBench 3.2.0; Fable 5.1; low","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:42Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:42Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c01c6fc889aa072b915a630bb9ac16584e369439b4955b980b73d07daf2f8cf8","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1","source_effort":"medium","source_model_version":"claude-fable-5-1_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"e8a4dc91b42d7f893f4a18b6b6fe531a9dfc9da8d9120206ba12d18a3130af04","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":50.91,"score_display":"50.9","source_stated_rank":6},"run_fingerprint":"edb60e6dbb45a9f4a113e2ff6674f822856c42cbb680480db87df0230a830ede","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e9a226f3cce5352633e4dfc5e6807740b830e699cd6efca9e6ea027e2b7cd034","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":11.196959,"date_is_proxy":true,"latency_seconds":1805.905,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.647},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":null,"source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5-1"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"05fb1054ce668b54b2b9defec8e43c6dd0d8228ab532960181c7ed547f7b8c83","result":{"confidence_high":99.88612,"confidence_low":81.66988,"sample_count":null,"score":90.778,"score_display":"90.8","source_stated_rank":6},"run_fingerprint":"6792e2451592e45a08ab84de549b6505dda485f3edeb32eee235a7cfeff98bbc","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"1d80e89cd26270ad49972961c8ac1190166f7dd402db3351554b8fbad29f304d","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.260604,"date_is_proxy":false,"latency_seconds":59.738,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-fable-5-1","source_locator":"Results · Overall accuracy","stderr":0.863},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"573dcfdb9c6de85172567d2cc6fea41f95848ab9725b1744c347393478cf76cd","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5-1"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"fac0930dafac67cf446d9f7a044fc4c271e30376b4caa8d1eb2f8ae2c9953e57","result":{"confidence_high":92.20648,"confidence_low":88.82352,"sample_count":null,"score":90.515,"score_display":"90.5","source_stated_rank":1},"run_fingerprint":"b872923ae6e035cfe5b6289375ccac973016a5492633ab400cbed647a99ea83f","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e3a88f326bc263e7fbae549fed922ea1f46f404b2f0fec1d85dd0aa1c514a568","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"46bd5aa62fe895c2ab9ac312f2373b544add5f949adf0f82443a32f739800cc0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.078703703703695,"score_display":"63.1","source_stated_rank":3},"run_fingerprint":"604e08f9778fc6cf0053a030f9f0294e4423f498d0ba04d9c32180d1f8f97087","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5025d17feef5cd23568cf517217a924dac64e84328803ec183e97fb4688421fd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"6f9f02ef4dd961fbc1db578037fed1a6128e87f5faa2df2bc09d601cc96f935d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.8796296296296,"score_display":"60.9","source_stated_rank":6},"run_fingerprint":"058f73b4c8c237330e419635dc59e337d4c5a1b8b0e730a1644158b8b84da59a","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"959d5487e05958d6001aa355568eaafc7ee8d0e62f1c6ddf95225af08aad8baa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"38a975f8c016bdca2d6637800994d224ae4812fa1823fdbc532f960ab38d6c7f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.63888888888889,"score_display":"57.6","source_stated_rank":19},"run_fingerprint":"4cacee15cd2e9729a7f52fd843eff8b97d9d6b6a5bc3d8011a18a165a364e5c9","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d4ec52ce97f8b48e41f81b30c80997f26e71ad96dd6b3ec8e3c66086b2fc1829","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"79df8de9338e44ed942440aa5ed1b6129981a10b2733696d75a4ecff886cd48f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.324074074074105,"score_display":"55.3","source_stated_rank":42},"run_fingerprint":"fc79fab41b0a443b4976c86973dd4a9fb7e7b9bd7a6c1fbc6ac2a55e54631bb1","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aa53ff2e35effb6e57d12d6d071055dd5f32f622585e7b948b4ee77303732342","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)","source_effort":null,"source_model_version":"claude-fable-5-1_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"c1526cbaf314d7d653ff0c8efca63e95b7b2fd17436136b009cdbd189d249781","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.671296296296305,"score_display":"55.7","source_stated_rank":37},"run_fingerprint":"0805cc3dd9e1f776342e39cc390f5288fb3bbdb7ab79ab18cc152fbece8e17b2","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e0ef79c81328b7ec5f53093f4f4ca98bf8dae26387f09b601d05d01832adfef6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"172c3b2629fabac12d6ef4df813564ff7aa893befed6d83e4eedbfbdc993fa0a","result":{"confidence_high":68.446065,"confidence_low":57.367025,"sample_count":288,"score":63.078704,"score_display":"63.1","source_stated_rank":null},"run_fingerprint":"2950b710b5a5f5857ff96f47f517cd2e5603a01852eac000e85a25217a7a8d62","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f145a00f2cf2c54d2029f2eec67f8434656be2604ac41c391073a7ca9e1bd0df","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-xhigh","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1dbcc459ebd8c43a02c70e83f548b861228f29f0323b203e4c5ad07105bcbd0c","result":{"confidence_high":66.337372,"confidence_low":55.135465,"sample_count":288,"score":60.87963,"score_display":"60.9","source_stated_rank":null},"run_fingerprint":"2d2ffc255ae760470a98d20c178081590d481603625564da9838b1a799c5728d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"98e4c25129897a6da0095ce300f2bb78e16a84d4ab5ae88d9f115828d6cc3b60","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-high","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"71d68fd77b79fa87c5b2dba4c554aa950c78cca8dd6e99287a4609741482d5d4","result":{"confidence_high":64.216905,"confidence_low":52.915677,"sample_count":288,"score":58.680556,"score_display":"58.7","source_stated_rank":null},"run_fingerprint":"54a590a6055058aa5ca740db14e8c48295167cce014f0ca941fcf0cdff755c99","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"22e179ecdeb83bf823e642c171efc3b170d3113c2a46cb6e008362662fc0f867","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-medium","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"acd23f169d3f7254c6053414533b15530190766d5baea837aa034477bb84b4d5","result":{"confidence_high":61.972454,"confidence_low":50.59144,"sample_count":288,"score":56.365741,"score_display":"56.4","source_stated_rank":null},"run_fingerprint":"6203b17b4fb5d923f2b1452fb8ce53d4af3a37a719a1a7508199e8cff5adcfba","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8ac4fd2c3fc08287b8252a4327a07a0f9fd2513776e7bc9975bfc4d62eccad92","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · scicode"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-low","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"6b8e0f8525594d6a93fa9e9b7659ccbcf5d2fbbb91d68cc7e34ec2d0ef19070d","result":{"confidence_high":62.30992,"confidence_low":50.939277,"sample_count":288,"score":56.712963,"score_display":"56.7","source_stated_rank":null},"run_fingerprint":"85d8d66320b6666e939955e363ef3057384bf691baed1ae94a77589868eb4c58","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:23Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f3cb4da280db7696b00ead7d79aef027453297d562161c6c1eb4317fac74b8a1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"2.294856","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-fable-5.1-high","source_effort":null,"source_model_version":"claude-fable-5-1_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"f88ae9977055b650bebbd66d326e915ad29134999b41ffa6ef9db480a625e519","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2143.2,"score_display":"2143.20","source_stated_rank":6},"run_fingerprint":"783ef43d16b88b1b4bce5e41f400f1596578b16c47e5c00c0f31ef38fbf09232","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a767f2975308af34c2c783223da6df5c6df58706e83c275bcc5b134ed2d6d678","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":33.367507,"date_is_proxy":true,"latency_seconds":3459.994,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.566},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5-1"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"334fa5b53dbc9f40db78dad3b772ba7a16d68a5944666ce472bd7461f1a7cc2b","result":{"confidence_high":93.33236000000001,"confidence_low":87.19364,"sample_count":null,"score":90.263,"score_display":"90.3","source_stated_rank":5},"run_fingerprint":"2f382fac2ada2c6647974c50d32ed159a298eb455bac12e2b7f7a3eaaed3cb84","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a36bcab24c1b4aab9647dc844d5881e4c73812c2f3da09948cd34928c77d9a07","metric_details":{"license":"Proprietary","variance":23.821063248121305},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-fable-5.1-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"17011532f457df0ba37cf11783ca8b78a52409f5003740e58855c4c5b020e596","result":{"confidence_high":1758.8095594459946,"confidence_low":1739.6776349706254,"sample_count":6318,"score":1749.2435972083101,"score_display":"1749","source_stated_rank":5},"run_fingerprint":"3779274adefe4d02e61472782c17d7aa31e023a1b2fe14260b0e8f9360838463","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9fabbb54a7163dfe2845d33b93c2b51c5d1645ed3f2910af4312b921c9613bdd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-fable-5-1_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"300dfb3a0b93c03d39fbc031f8239fc6f3dc5cdb27b776c68b8e4a85d1ef658f","result":{"confidence_high":1771.65,"confidence_low":1744.44,"sample_count":3036,"score":1758.05,"score_display":"1758.05","source_stated_rank":5},"run_fingerprint":"ca910672caf407e2e170cd294731635892f0d50dc1eaf9c9cc8042eaeac3294a","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"mirrorcode","metric":"mean_score","name":"MirrorCode","release_id":"release_07bf24088c0805f7b6df7a88","split_or_window":"15-programs-30-language-tasks","unit":"percent","version":"ML +Private 2L"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":25,"at":"2026-09-10T13:19:30.449Z","basis":"evaluation_started_at","evaluated_at":"2026-09-10T13:19:30.449Z","first_observed_at":"2026-09-10T16:33:47Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ad793cd6313e33632cf591286529326ba1909771f5cf459882cd4636e95ffe00","metric_details":{"best_score_across_scorers":"0.733","model_release_date":"2026-09-01","stderr":9.17},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mirrorcode-ml-2l-claude-fable-5-1","log_viewer":"","logs":""},"run_count":3,"scaffold":"Epoch AI Inspect ReAct agent","tools_allowed":"shell, text editor and visible-test evaluator; no internet"},"protocol_fingerprint":"354d57ef4adf59542f1b1d54595b2a80d597cf3c60ce6b9b083a27c07ae6f6e2","result":{"confidence_high":91.2732,"confidence_low":55.3268,"sample_count":30,"score":73.3,"score_display":"73.3","source_stated_rank":2},"run_fingerprint":"397daa90c50fe74199b341389b47c16f2ddca82fc57a61157dc83731490fbf23","source":{"content_hash":"3e212e97e03d0343e676ecc7ff046ee7b94d219ba7d863364634b7361dc89b4b","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://epoch.ai/benchmarks/mirrorcode","id":"epoch-mirrorcode","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · MirrorCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/mirrorcode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"08821f07d6d927a86387f354b6df52c0eab52d2a77bf9f33a4a5ed107b5c14ca","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-fable-5-1_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"34530ceafaa84fb9bc61398ac7f4834324e260c9d0882ed93f9fe0aa2f2f6843","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.9,"score_display":"92.9","source_stated_rank":3},"run_fingerprint":"a1bb484fc42cc14098d979a57285926a0b52e428ae435629dc6ebaf1f3fa356e","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cce80dca28a409fac412610bceed4555ec9685dab1323479ba21538c04383317","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-fable-5-1_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"1a6c0151f3259a95aa321cc4aa9391fd63e2b5464f5f596220e864f76bb4bf44","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.30000000000001,"score_display":"92.3","source_stated_rank":5},"run_fingerprint":"77bd91320110d967e5eda34fb3a8a3a494ae42fe06a92ba28342ac66fb9a6b18","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6b16a65bd5b673cd8901cdf5b426dfd142b1e33f7a1d47cdcc238f056cecab74","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":3.074796,"date_is_proxy":true,"latency_seconds":500.355,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.991},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5-1"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"39d9e2521e64b9001cbc7d3613d17c165a5e00ea5f1914706b8823dbc019520c","result":{"confidence_high":86.96136,"confidence_low":83.07664000000001,"sample_count":null,"score":85.019,"score_display":"85.0","source_stated_rank":4},"run_fingerprint":"19915ca28878f04cee295fc4a7e831f27c31d048bee9ac1e5c62351f29cdd47c","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fb8a2952763d01ccc5c95fd79f55f89cdc9a79c9d4e91378553223d31cc4416e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · terminalbenchV21"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5-1","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"d9ed243d64a064f393f15ea99d74de2d061524b41043479c1fafc016d3268bd2","result":{"confidence_high":95.632002,"confidence_low":83.714613,"sample_count":89,"score":91.385768,"score_display":"91.4","source_stated_rank":null},"run_fingerprint":"5df3609887a91136dfcdfb612926612992e9393872c68ad1c4427e35d75a9e41","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a3f5a746051759b0aa670e5da88cc49e365d3a2688208f46a5be711268500033","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · terminalbenchV21"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5-1-xhigh","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"4d8febed75bda2a1175352324425a7a6fa8ebbd1ec845db1b14d4668eed93429","result":{"confidence_high":95.374812,"confidence_low":83.253733,"sample_count":89,"score":91.011236,"score_display":"91.0","source_stated_rank":null},"run_fingerprint":"dd3d92891a132bde3f637c19e7465242f550b8c85a219510eb280bdb84ded5c3","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ff9c4b6ea5a65a98996b2716cb7a1983b1cf5e339a51316937d37cdb88f62c42","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · terminalbenchV21"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5-1-high","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"f0a217a1f4a1a4719f90b48d145dee1bfbb25b0cf5ce640e89ee623399ba8585","result":{"confidence_high":94.588206,"confidence_low":81.886132,"sample_count":89,"score":89.88764,"score_display":"89.9","source_stated_rank":null},"run_fingerprint":"419fa5090d54c3cd563b075daa97ef0b6741a192e7baf2422274117ecbea3363","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aaa842f001aab0d69a4d93f3a475e7523f7ef218af158581e727e20bb8106aa5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · terminalbenchV21"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5-1-medium","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"ad15d5f55d0bbe3ac37955f923eeadbcbb39635c8e80654e6c6d87fc5fe53ff7","result":{"confidence_high":93.233339,"confidence_low":79.650655,"sample_count":89,"score":88.014981,"score_display":"88.0","source_stated_rank":null},"run_fingerprint":"995504eaf0a288c775d1057de471a22892632b116d39af793d1d05e31b5aca34","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d3ee45320287fafaaa764736b07766885df039d6fecdeef2010c425fdd394a30","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · terminalbenchV21"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5-1-low","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"85460d6fa04237411265058c27d3f59b816128a4e744c12aba64621924d34aa5","result":{"confidence_high":90.97258,"confidence_low":76.166864,"sample_count":89,"score":85.018727,"score_display":"85.0","source_stated_rank":null},"run_fingerprint":"30611e4a5b0974099e3249bdb4ddc87c49b074a130cc21ebb3cdbae319a65d7e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6d66ff55084548fd7d6bb6050c93a1dd524df27ba4e781979039400b7b1414c7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · tauBanking"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"ec3b1edaa173065f91288781da004d030be44710ec3f032eac591b26baeefaea","result":{"confidence_high":57.066997,"confidence_low":37.578071,"sample_count":97,"score":47.216495,"score_display":"47.2","source_stated_rank":null},"run_fingerprint":"696474e69415cf43289ecb38bb7efdcec7c3ec2e5ac3fb694a85807afed2bccd","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1fcb43120fb7e55c1fef065247eb96c85224718e31d63dd16d171f8eb0ca15bf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · tauBanking"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-xhigh","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"f86bef774b925afeac5a14d87391e9351b79e97bf45d7c41874847d7e04842e7","result":{"confidence_high":55.659638,"confidence_low":36.208798,"sample_count":97,"score":45.773196,"score_display":"45.8","source_stated_rank":null},"run_fingerprint":"5e364c315839cf4b7291d7e4139ae6876d9528e076f6b11792ebb79d3e0906df","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2e28f1bf265d80dbe76c840085b2f3990b656218d13745d476923e39afc78841","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · tauBanking"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-high","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"24b636f0b796702386061ce2ab21d4f51f30b58bb1e0808a15d80a964d16b14e","result":{"confidence_high":53.02495,"confidence_low":33.686884,"sample_count":97,"score":43.092784,"score_display":"43.1","source_stated_rank":null},"run_fingerprint":"ff41b6b7e6a5c881d07e240617a7570cb80804eac9b6d1aa39b2a94a9ef1ef63","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9861c30e4f16598af1ad9c79db3dbdcba7d379a36c05ff3ec79cb75cee8e8b1b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · tauBanking"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-medium","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"bd02f0f55d262591e17d04ddf593b5e5351d0c8268c92ee9261b8c6149181e96","result":{"confidence_high":50.979411,"confidence_low":31.765805,"sample_count":97,"score":41.030928,"score_display":"41.0","source_stated_rank":null},"run_fingerprint":"814dbc6f129752afa060980c4a220aaa210d0bc936e704f69774c9f95b884f1b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c198bc0929b2f7b89159e3fa1bd041ba51ee5719b7e90d630ab8e970821f3717","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · tauBanking"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"claude-fable-5-1-low","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"30e6ae3c829b2c39e1aa8c5af46ba4a246b62bda2bd1689afb7d47dc65bd6051","result":{"confidence_high":48.917131,"confidence_low":29.861468,"sample_count":97,"score":38.969072,"score_display":"39.0","source_stated_rank":null},"run_fingerprint":"74994a1c1aa0574b9c00641fed9f1a624c2af6bc4e7215c9d57002866ef176b4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0f9c5730c25b6d0769d7b3e5c53845e61b9c7f761d5b45fcde92f9c2222550df","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.718","mean_standard_error":"4.6","model_release_date":"2026-09-01","notes":null,"standard_error_points":560.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5.1 (High)","source_effort":null,"source_model_version":"claude-fable-5-1_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"389e353be55ea4ebf1208f1346ba78389965b6981679c8bcd7418e4c4111ba9b","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":59.70000000000001,"score_display":"59.7","source_stated_rank":11},"run_fingerprint":"52b6446b4533b099576689ad7037717c00443ce85e65ccdd9526a871101341b3","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b37b8c3c382366017c7605e9861a6b29058d4439da4e7fe89e5793c7c71ed371","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.7709999999999999","mean_standard_error":"4.3","model_release_date":"2026-09-01","notes":null,"standard_error_points":490.00000000000006,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5.1 [claude-fable-5.1]","source_effort":null,"source_model_version":"claude-fable-5-1_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"f16d13fb0b1e177319de48344fbbcd840c0c5daba70ff10f0de838e381712c6f","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":68.6,"score_display":"68.6","source_stated_rank":3},"run_fingerprint":"c121c74a00b3c13a1ad1ecf9d148d9a713c184d62fd64dbb699da03b1e176f9e","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5426876672d785844ac12d32","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_id":"epoch-apex-agents","source_label":"reported","thinking_strategy":"adaptive"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"40a12b281d618071b64d1daad1565a61db2518d6b1391c1287f9f606c081f172","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":15130},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Claude Fable 5.1 (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1267be1de0a0c2a16329519adc670503937c4db4f51f52f25c2bb3ea0a44f582","result":{"confidence_high":0.16205588648253044,"confidence_low":0.12415154304102166,"sample_count":1683381,"score":0.14310371476177605,"score_display":"0.1431","source_stated_rank":1},"run_fingerprint":"9b3f93e4867a88f609bd91a11bc7e45d800b27e342fe57e11864d00ec356d873","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":28,"at":"2026-09-06T19:02:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T19:02:23Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"59c48de94a831bb9e25313d7439c4a4f68ff91ce4cf7220a10aa0b3ed726919f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5.1","source_effort":null,"source_model_version":"claude-fable-5-1_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"93c8c86cf424031291284c60fca053346f36cfd3a5e4fbe6d612ffe5a6219bd2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5421.556666666666,"score_display":"5421.56","source_stated_rank":23},"run_fingerprint":"d5f8cac6a332aecdf89a2f3007cf45b92cffb7f59c029742aa7e75dd9022624c","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6dd039f109f0800f20553ee1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_id":"epoch-vending-bench-2","source_label":"reported","thinking_strategy":"adaptive"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"af35210eb51c4cd58ff7b75ab1c3f91a5029538c25cc0d70ff74a686866d1848","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-fable-5-1","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"b846beb80a2752350cbd9f4912375c4a60015e47f5ce0b7dc3d7fef86f8214c7","result":{"confidence_high":1759.25,"confidence_low":1710.09,"sample_count":220,"score":1734.67,"score_display":"1735","source_stated_rank":null},"run_fingerprint":"0114a7f5dbf0b012e82de35eb007fd1225ce03d4a5202a788a73b782f542d810","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a591206608ea874c100731695469e0849b5f1740c686a30940a0f8a00a6e4530","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-fable-5-1-xhigh","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"7b94283ab9ecc5ccfd1c458f9c30a6c07bfd28665146313d7036c1b23d7b875c","result":{"confidence_high":1746.08,"confidence_low":1695.77,"sample_count":220,"score":1720.92,"score_display":"1721","source_stated_rank":null},"run_fingerprint":"636e5e5d3e877b0e4668dda8cdf6fd159ba7833ba2808bb1ae73f7b90e4fe0e1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"82b4015bebbd932f59c24c4a6851aa37cbdb9cb7ee763810bc624bd9810d4793","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-fable-5-1-high","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"34c0434a27ada20802cb469f98f2cf28f288c0e8687c2bbfa5b70f62797b208b","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1617.29,"score_display":"1617","source_stated_rank":null},"run_fingerprint":"35ef645b98f0df54817142d12cc43eff0c2d3fe369ad109561032376d95c56f9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a11908c325c99f9910416bf29c7bbbd33ea392c37d7bd19138c520553533d8b3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-fable-5-1-medium","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"6e691ca8d73bd386cc7bf5424a4c8b61055f934bf95e04f8d5c131c4271060f9","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1535.85,"score_display":"1536","source_stated_rank":null},"run_fingerprint":"cc4cfb1ec13401e99eed83d7f5beed87c55737d1afb884c0a2a2f06b55cb35f9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"317b04189b55e4385738da92c86ccf27eef1b3377d408881647148ed5afde078","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-01","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · gdpval"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-fable-5-1-low","upstream_model_label":"Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"da7d0e9dd46f33351f06ca903c78cca7c2076a011b24f48378111cbcaf6243d5","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1449.57,"score_display":"1450","source_stated_rank":null},"run_fingerprint":"575c016896367081cea6690600880fc16e10444d33fe2c768b8754c81c03bf37","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-4","metric":"accuracy","name":"Terminal-Bench 4.0","release_id":"release_a2b8a5b379df93e681fb82fa","split_or_window":"4.0-provider-run","unit":"percent","version":"Terminal-Bench 4.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"21b62c8d3ee2b966d3dcb31d709f27865ac204be8b797877f102547a23f49388","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":19.5,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; max"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; max effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"max","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"61d352ece2a09a66676b073de638b881999de9d0f42bb781dfc8d6b5fb702ac2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.8,"score_display":"55.8","source_stated_rank":null},"run_fingerprint":"c75c292501b463ba82602e7753d74957e45813596ce7d72ef0a5962e9d05b074","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; max","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-4","metric":"accuracy","name":"Terminal-Bench 4.0","release_id":"release_a2b8a5b379df93e681fb82fa","split_or_window":"4.0-provider-run","unit":"percent","version":"Terminal-Bench 4.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"a056f97a942048e19cb17b4f9b7e9df110d5fe912eb795639ad545a6c5e323f5","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":15.8,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; xhigh"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; xhigh effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"xhigh","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"8d7b15c64ce57ab53853b78c614e669842ad6f4b5ab66a7d666ef3ae658671a3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.3,"score_display":"51.3","source_stated_rank":null},"run_fingerprint":"be449a3b9451c9bb2b13fd3b6d223c2dbe1dffad0d5e26f44b29e807c6eeebc5","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; xhigh","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-4","metric":"accuracy","name":"Terminal-Bench 4.0","release_id":"release_a2b8a5b379df93e681fb82fa","split_or_window":"4.0-provider-run","unit":"percent","version":"Terminal-Bench 4.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"cc5fcc23f9b6b180747ec63178132fd5d6a0a17768d9e59948821250ced60047","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":10.5,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; high"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; high effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"high","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"baa04ee2d485cc6050808e23d82006e40c013d0e7075d51914b049f357543175","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.4,"score_display":"49.4","source_stated_rank":null},"run_fingerprint":"d1a27178f380aef0c6bbc1533e8d723d34a0ae0182a797e5988548e1b29145ef","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; high","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-4","metric":"accuracy","name":"Terminal-Bench 4.0","release_id":"release_a2b8a5b379df93e681fb82fa","split_or_window":"4.0-provider-run","unit":"percent","version":"Terminal-Bench 4.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"066ecf1b7442a114dfd9a36cd6bd78c0c670addcb0a830ab7a34a3d980c1890a","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":7.8,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; medium"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; medium effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"medium","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"dd8a16bc7e199add203f9b00e5eb7bc957211b72c9298bc761a5410af50d2d1e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.4,"score_display":"43.4","source_stated_rank":null},"run_fingerprint":"6e3e1171eaa7bd40cbaede30d85047a374065ec0cc71a3682cc1b2f5c35813ef","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; medium","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-4","metric":"accuracy","name":"Terminal-Bench 4.0","release_id":"release_a2b8a5b379df93e681fb82fa","split_or_window":"4.0-provider-run","unit":"percent","version":"Terminal-Bench 4.0 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"f087593a258e10454c1f3aa7b1e2638d729354af72bd08a0e58c22678c6f0d4c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":5.7,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; low"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; low effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"low","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"10d6058077a33e27cf1e4e38f40ca6b384acc59847e52305dbded1e4a4b52ab1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.2,"score_display":"40.2","source_stated_rank":null},"run_fingerprint":"ce91c24e18c7bb44e416dd75f4bf2da19bb56c7a74185f1cf8311a44b6c41380","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench 4.0; Fable 5.1; low","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-science-0.1","metric":"accuracy","name":"Terminal-Bench-Science 0.1","release_id":"release_cd2782e877731fce64a82910","split_or_window":"0.1-provider-run","unit":"percent","version":"Terminal-Bench-Science 0.1 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"6c80ea8e28bc43c74229f515c869d9b516947673c3e37a81ca48c70ca4093e9c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":37.9,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; max"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; max effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"max","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"b47f60271af319570cf44906df415f8066390768abf04e0651427411def0979b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.6,"score_display":"52.6","source_stated_rank":null},"run_fingerprint":"a213f0b8b766be80cd3dac1305705ca163d8cf8e0e9f6f391dafcf66feab1389","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; max","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ccb7cfb699f3e49429f43ebf","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-science-0.1","metric":"accuracy","name":"Terminal-Bench-Science 0.1","release_id":"release_cd2782e877731fce64a82910","split_or_window":"0.1-provider-run","unit":"percent","version":"Terminal-Bench-Science 0.1 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"f90b795da81fc38c59ea4d42282526f34e06913dc24e13295f959850d6bb6e2c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":31.8,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; xhigh"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; xhigh effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"xhigh","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"4cc5d9e69586c0de111490a3586749283e5786acd9c499c53f136cd452f90a6a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.5,"score_display":"49.5","source_stated_rank":null},"run_fingerprint":"9f64dd8ec769c538259b305d103075a3ea468380852d3bb713802f34ffad70ca","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; xhigh","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b672f752d3f34b6d6a8558f1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-science-0.1","metric":"accuracy","name":"Terminal-Bench-Science 0.1","release_id":"release_cd2782e877731fce64a82910","split_or_window":"0.1-provider-run","unit":"percent","version":"Terminal-Bench-Science 0.1 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"883daf7b6ed57eb8d02369126646f50813b7fd0fd2fda2e2766559341ce78cdf","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":20.3,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; high"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; high effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"high","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"80d828323d9670a343471e965942de9320480fcd84d5864c0f098c64f66cb012","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.0,"score_display":"40.0","source_stated_rank":null},"run_fingerprint":"d009a3fd57ddc10dae410d8924b3cf22ecad4e4c7b6e4e9ca92f77b711bd5a10","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; high","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_0e5a1af41c411d83363a781e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-science-0.1","metric":"accuracy","name":"Terminal-Bench-Science 0.1","release_id":"release_cd2782e877731fce64a82910","split_or_window":"0.1-provider-run","unit":"percent","version":"Terminal-Bench-Science 0.1 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"fd788e6ada1cccf04f5e9be512f9f422205ae0631ff2bcb0f784866887b7ed94","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":14.9,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; medium"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; medium effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"medium","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"c3bd5f7e0cb2b5eb4a7540d63d5d340424a0724bd2fb23655fbe43c2215ea08a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.7,"score_display":"35.7","source_stated_rank":null},"run_fingerprint":"5337a923625ddab8c8ac04b8cf5b06fb46be9265e52793cd896384b13b15fecf","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; medium","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_74602224514915764af40888","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-science-0.1","metric":"accuracy","name":"Terminal-Bench-Science 0.1","release_id":"release_cd2782e877731fce64a82910","split_or_window":"0.1-provider-run","unit":"percent","version":"Terminal-Bench-Science 0.1 · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"0a8ffdab5af6c8c3f3cfa0b2e1ce23d418434f9926185324d7bc1069a735c037","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","cost_per_task_usd":11.1,"evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; low"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch evaluation; adaptive thinking; low effort; production safeguards enabled","provider_default_effort":"high","reasoning_effort":"low","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"terminal tools"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal tools"},"protocol_fingerprint":"9845f8dd873c6b12f42102f84603c3ac3fb3fafe632ed27bddbce82a0ec10fab","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":26.3,"score_display":"26.3","source_stated_rank":null},"run_fingerprint":"288cfe09d7f9276bb39f99e5a5081b2f2664d4d54b1613c665f2cbb49adbac73","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Accuracy-vs-cost chart; Terminal-Bench-Science 0.1; Fable 5.1; low","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6a0e335cc30266058d475ad0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"success_rate","name":"OSWorld 2.0","release_id":"release_e7bb52b116c9e290ad684e60","split_or_window":"august-2026-partial","unit":"percent","version":"OSWorld 2.0 · August 2026 task release"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"8b43ae1653e6cdebd93f094dce6ad764a0733b3ba7cbcc3c93dbccf2879d68f1","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Launch comparison table; Computer use; OSWorld 2.0; partial; Fable 5.1"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","osworld_partial_not_canonical_strict"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch comparison table; production safeguards enabled; exact effort and run count not reported","provider_default_effort":"high","reasoning_effort":"reported","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"11c5b8da7d1625ecf3af992e92a5a4e6d66940a641dc52fb3ea9dfa50428f88d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.9,"score_display":"77.9","source_stated_rank":null},"run_fingerprint":"833c2398e1d964938827187b07cf5459f9615f30633f4d5a5bd8e7af036d8667","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Launch comparison table; Computer use; OSWorld 2.0; partial; Fable 5.1","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c52c1ffeca7e4a640a86e739","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_id":"anthropic-fable-5-1-launch","source_label":"reported","thinking_strategy":"adaptive"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"success_rate","name":"OSWorld 2.0","release_id":"release_9bce35e4d0ce0a517fa12ed8","split_or_window":"august-2026-strict","unit":"percent","version":"OSWorld 2.0 · August 2026 task release"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"bd22f202ad9d55904542db2d9040a201b69839b90763e47324addfaf6f37c1fe","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Launch comparison table; Computer use; OSWorld 2.0; strict; Fable 5.1"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch comparison table; production safeguards enabled; exact effort and run count not reported","provider_default_effort":"high","reasoning_effort":"reported","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"b36bf4f6fa5f1c544b309900c69e23704f81ba899237f3fcf8431fe3b051d8e4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.7,"score_display":"41.7","source_stated_rank":null},"run_fingerprint":"3e8f67c15fab54ed779c80e9485433ba3e5d5beede32d61fe80d9ff705f8beb3","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Launch comparison table; Computer use; OSWorld 2.0; strict; Fable 5.1","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c52c1ffeca7e4a640a86e739","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_id":"anthropic-fable-5-1-launch","source_label":"reported","thinking_strategy":"adaptive"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"automationbench","metric":"success_rate","name":"AutomationBench","release_id":"release_587f0dd1273360ff4feb0c7f","split_or_window":"provider-run","unit":"percent","version":"AutomationBench · Anthropic launch evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T20:50:42Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-01","status":"fresh"},"measurement_id":"dedad92829a77f50ab1f77b5ace815f589a16a499d3fa805a4fff0a9e988c52c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","model_release_date":"2026-09-01","source_locator":"Launch comparison table; Business workflows; AutomationBench; Fable 5.1"},"model":{"id":"anthropic/claude-fable-5-1","name":"Claude Fable 5.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["vendor_source_specific_not_cross_model_comparable"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic launch comparison table; production safeguards enabled; exact effort and run count not reported","provider_default_effort":"high","reasoning_effort":"reported","result_published_at":"2026-09-01","source_content_hash_method":"anthropic_fable_5_1_evidence_v1","source_content_sha256":"03b8e240ad244a15829d753765a1fd2fa5ce0064b7ed033e7b526b5f4225dc9a","source_published_at":"2026-09-01","source_published_at_evidence":"The launch page states September 2026, the exact model document states a September 1, 2026 release, and the reviewed launch artifact was first observed on September 1, 2026. Their intersection fixes the first-publication day to 2026-09-01.","source_published_at_precision":"day-derived","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"084f67c3372e48b8119eeefcec7afc736f1e0d48febf04de4be378c047ba9c45","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.4,"score_display":"31.4","source_stated_rank":null},"run_fingerprint":"cd36c72be2bd4128f9c213c5900fcf758ac60f5f120dc446c7738b51a3b1a1f2","source":{"content_hash":"c65f4cdb68ec3cf6656ceb9f2ff1116187dd2dee981cd63484daca08cd95fb10","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-02T12:19:36Z","homepage_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1","id":"anthropic-fable-5-1-launch","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Launch comparison table; Business workflows; AutomationBench; Fable 5.1","name":"Anthropic · Claude Fable 5.1 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.anthropic.com/claude-fable-and-mythos-5-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c52c1ffeca7e4a640a86e739","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/models/fable-5-1/overview","source_id":"anthropic-fable-5-1-launch","source_label":"reported","thinking_strategy":"adaptive"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"235ae8fb34a58cbd24035944a372320aba4d36a2442d2a2feab14af334d33526","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-24"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Opus 5; model release: 2026-07-24","source_accessibility":"API access","source_model_name":"Claude Opus 5","source_model_release_date":"2026-07-24","source_model_versions":["claude-opus-5","claude-opus-5_low","claude-opus-5_max","claude-opus-5_unknown"],"source_reasoning_efforts":["low","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3553452d1de18a059f073e12769ab400dfc5c79df8927d9ad22096c123cb525e","result":{"confidence_high":166.7,"confidence_low":160.2,"sample_count":null,"score":162.94,"score_display":"162.94","source_stated_rank":5},"run_fingerprint":"5e068368a1f45e6e8641a016cf35973dab5e1892ebdbfa8610d070a4e3a9bfc6","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_564261edace44339c4aa1271","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8acedf6351ecbac28c1296a3a592f96ea141c7905ab00b956fb994689159fe63","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Max Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"5f859a97254123fd9f97c0cbc1db3b339b6d76a86b7d7df11ac6622e510c1cf5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.700187,"score_display":"50.7","source_stated_rank":null},"run_fingerprint":"8c8653e383f071de37cb45a8073f83f81f7a28fd3aa7ccfa6618b256e10e2faf","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5a0a0eee7bf827a54372284b006b5985dda98f7aa337c9fc0f5150f4429c03ec","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-xhigh","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"8a0a40809782a8514778b684e092e86c6b00cd8556bda9f44aa7c65cbf525997","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.651455,"score_display":"49.7","source_stated_rank":null},"run_fingerprint":"576ada136114ad754a77a22bfee5fdcd1a096744002c5af490901a3482f4f94e","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"344d4b8d2f866a8d17e27771c9abe6c87a502e55deded167d88550f4759e1003","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-high","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, High Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"94b0292933f530231eb566131afda9b089d4b61fe47b97738e36166d2f5e5a5a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.237193,"score_display":"48.2","source_stated_rank":null},"run_fingerprint":"74104af0ba2dde7b3df8c87372f198085b517c77adfe606d5ae882d26bbbeae8","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"02870267a6eae8369fbbadee809279009e7ca53f324a8326807d08f06b5ab61b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-medium","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"ce5260fa3f889e2de8c08d1c17a544961aeef2a73094d901fb0c8a514ad110ce","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.057577,"score_display":"45.1","source_stated_rank":null},"run_fingerprint":"7faa45acf4c31e3547c5acaa0ac4b9cdffd57a7c7a73e81f42eefdd55ca528a2","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"021dec232694fb5dea6baf3f8d2b552d858ff76651b1ef8f51e9893e58d5571b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-low","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Low Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"2c2c7269fc27a6e49867bb5230a4176273e850eab178c6d69989717674ff8132","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.788336,"score_display":"39.8","source_stated_rank":null},"run_fingerprint":"3a6bab34cc3c770b584a9475fd44e1727e14fcbce3b87f080b3bfd2290cb77c3","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6a7940ce4bd04196c1963392d7e9af47f538c43402f3d59ddb0769fc974666db","metric_details":{"license":"Proprietary","variance":6.409438228170221},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1494.4560168616663,"confidence_low":1484.5319868300112,"sample_count":28937,"score":1489.4940018458387,"score_display":"1489","source_stated_rank":14},"run_fingerprint":"8d6d0f9acb040cecd5de397e01e46ff6143b2b6947a5e8484c33f59d396e05e9","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d5e0406b6e49764bda02abda8d2772b6fa0c5d6c01773837bb89ee94277a7f67","metric_details":{"license":"Proprietary","variance":3.767155056051918},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1493.5482886004593,"confidence_low":1485.9400375480839,"sample_count":59482,"score":1489.7441630742715,"score_display":"1490","source_stated_rank":13},"run_fingerprint":"e4c2d73a6c2e73933e7247b8efdb301ce259ae61cfbf1c5ab9a33e4cddc43004","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"65d2ea3311ee8db8fd952316491612aaacc8b1299f3c1badb13ed6e5d219199a","metric_details":{"category_scores":{"Agentic Coding":65.202,"Coding":81.4455,"Data Analysis":74.55033333333334,"IF":63.76650000000001,"Language":88.68799999999999,"Mathematics":95.731,"Reasoning":91.2115}},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.08497619047618,"score_display":"80.08","source_stated_rank":12},"run_fingerprint":"ed1540d074e358c984aa850c775bbcf9d929947aac2c8a7fae8397accb93a556","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"5b1803aa53ae37b78e775ac27f61a658a307f6e77b3e139dc08d064e7ad296fd","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.B, p.162"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1d000b8ca17f30a20519c0fbea4c0033b69f0c618060fec604365eddb311a0c7","result":{"confidence_high":58.233245,"confidence_low":54.347423,"sample_count":2500,"score":56.3,"score_display":"56.3","source_stated_rank":null},"run_fingerprint":"5a69b22b38dbddbd17a3674fb69aad64338d3e3f2220461284fccd7ce033265d","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.B, p.162","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_54c2373720fce61d7d0d3fab","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"51696f2207e5275cd9c94ca8540288646b7263df173ae89decb4756c1b8f22bf","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.B, p.162"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; xhigh effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"xhigh","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6a5710f5c383e3426c6dddb2a9ecd2e1ae90d7ba7b8faebaa436756ef8f3fe4e","result":{"confidence_high":58.332591,"confidence_low":54.44777,"sample_count":2500,"score":56.4,"score_display":"56.4","source_stated_rank":null},"run_fingerprint":"f9ef71c7789a774dce7f5e9729113f3189aef08493824103b3064269264743ad","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.B, p.162","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1353d78f98b40c727e90d9d9","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"2844c1699fc159538d3e56dbfb0fa53f1f1fbe63f703459766db492dcea5cc2c","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.B, p.162"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; high effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"high","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4aaeade1ec8fb4a1d2b21f4807f2ee81d72ffa1072dc44e15aefb4fc123ca015","result":{"confidence_high":57.93516,"confidence_low":54.046429,"sample_count":2500,"score":56.0,"score_display":"56.0","source_stated_rank":null},"run_fingerprint":"78c0e248bc76fbd6769320759686ef1d6438484de7f07f3c6fdadd83396800b0","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.B, p.162","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_6a41327fb38e9b09fe8603d7","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"b22d21b3008eeb9c59fe180cb34f469d8e92dbf904ef1437ebd5e387937adf7a","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.B, p.162"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; medium effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"medium","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b75fad39d6ec7897056b525655375085920c40d350e7e6fe25617ac166bfbf64","result":{"confidence_high":56.145141,"confidence_low":52.241971,"sample_count":2500,"score":54.2,"score_display":"54.2","source_stated_rank":null},"run_fingerprint":"9d7cc592f494ebb9e9a47c0756d2a169894ef0c5663a094f7732d25936499e8c","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.B, p.162","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_db766cc6c7c38411e11456f3","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"fe187b2cae75f120ac06306f604f3906ead120e6b483338d242892db07548d1c","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.B, p.162"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; low effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"low","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"61de9291152845ce086ae53cbf66f41254e9d9187c4ad795684736ebdebaf92a","result":{"confidence_high":49.759977,"confidence_low":45.846773,"sample_count":2500,"score":47.8,"score_display":"47.8","source_stated_rank":null},"run_fingerprint":"685264f7d00da19be34df5627672d11301eac58bc3cc7a1f9d828f147962435f","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.B, p.162","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_32743e251ec4b9ca8df83e83","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"evaluation_at","evaluated_at":"2026-07-24","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"868ec6c69e7fcd28f0837bb098228265adc81ac187287294b1d0cb0d4d4cf62f","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/anthropic-claude-opus-5","source_model":"Claude Opus 5"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f01ecd83570b25385fbf939295db1088a3831bbdf4d5a92cdd96fff3ca441290","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.3,"score_display":"56.3","source_stated_rank":2},"run_fingerprint":"a7c9d9a0886a815912b4d742c843f3d927c3551123451e30abed162b7c175b1b","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fe0df0076c8d45add731bcce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"39d40f1527f70b2f0a53ebf8c84301b9d2dd7d80d25599cb1ec5bb02e3528eb3","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.A, p.161"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"b0d27d4ca635465399922319ffa39f25149d5c5eaa4764a86b6a08c4d9c6d0ea","result":{"confidence_high":66.549523,"confidence_low":62.805369,"sample_count":2500,"score":64.7,"score_display":"64.7","source_stated_rank":null},"run_fingerprint":"30608ef7190612b2ac8729a51b4035e8db6bbf883e2a2d21f654f96cafbc840b","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.A, p.161","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_54c2373720fce61d7d0d3fab","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"597fa6a4b6c26626f96c7cff9cd65e052a6fe8d2a7c9cc0c209c60a4da56a47a","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.A, p.161"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; xhigh effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"xhigh","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"af15d2ece8ca092471e4f29a4e20a201ffe8f60c21065961fb85b21a823e0ff4","result":{"confidence_high":66.648162,"confidence_low":62.906423,"sample_count":2500,"score":64.8,"score_display":"64.8","source_stated_rank":null},"run_fingerprint":"75941eede58a120db2595bb416fab613023db0815013874d146d56d494cb58b7","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.A, p.161","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1353d78f98b40c727e90d9d9","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"23a04db3557b635090c44b72dc2ae4a6dc6da03d2aa00a2b55b9a74b91fea5e9","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.A, p.161"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; high effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"high","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"52d6c0c23c79a1dc2fde280259d08c0817f045e561dcf1d7137bdee45b299c6a","result":{"confidence_high":65.06887,"confidence_low":61.290625,"sample_count":2500,"score":63.2,"score_display":"63.2","source_stated_rank":null},"run_fingerprint":"9d2be4a9d8a931c7726383bd7ae6135257704f55bf76855f2ab9b6ddfa700aff","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.A, p.161","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_6a41327fb38e9b09fe8603d7","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"6767ff43a28de996a059dba5946b1070a21a064f92b8a5a25ac8081ed7ba8d76","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.A, p.161"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; medium effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"medium","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"cc117f37973110dc8481880ec2a3b880190043027eef96d6f3abb7a28e81113d","result":{"confidence_high":63.190566,"confidence_low":59.37476,"sample_count":2500,"score":61.3,"score_display":"61.3","source_stated_rank":null},"run_fingerprint":"a70b40479557b00cce18d256973333826825fb05ded6ad652739a278da4d3ab8","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.A, p.161","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_db766cc6c7c38411e11456f3","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"ae3acd9c3a4567ac44d29042a5f5391d432f15c40ac50ec4a6e8b16f14494ede","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.10.1.A, p.161"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; low effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"low","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"75a92da7eb339485547f28eade4311b935ac50c8d231533fcbce2af259eb5c75","result":{"confidence_high":58.034529,"confidence_low":54.146752,"sample_count":2500,"score":56.1,"score_display":"56.1","source_stated_rank":null},"run_fingerprint":"a782249749729659a143e9c70032c22c65aab8d534d97adf12109e963b2c0bf9","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.10.1.A, p.161","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_32743e251ec4b9ca8df83e83","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9ae674f94137dd87122705191466ca461ca0860bf2f136c61b832ecae5108c52","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · hle"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Max Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1e81256caf684d8eec577c6a8080a1e254f3004233c6bd143c67daeccd176ec4","result":{"confidence_high":56.95471,"confidence_low":52.75923,"sample_count":2158,"score":54.865616,"score_display":"54.9","source_stated_rank":null},"run_fingerprint":"b3ae8bea66dc042b2a7b0e1933fa4b35b469026c406e585cd0477dcfd9e94013","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"caa408bba1629831c6b15401d85187456a608d7ecf5d05ebc20bb3f904d00c6e","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · hle"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-xhigh","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a227302f3a27912b9055aa187e763776b1509e9a2f745ce9a2ede7c95d355f80","result":{"confidence_high":56.493956,"confidence_low":52.294846,"sample_count":2158,"score":54.402224,"score_display":"54.4","source_stated_rank":null},"run_fingerprint":"b9cafdf38819fe52feba1a255ac7283a435b444e15bc9289a88222e6d4ead6ba","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"667a0fd2fb68dbdc7c71eb2dcdd5af4117589b168ff85359e2e5b6da85dff568","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · hle"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-high","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, High Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8c3a42f4d09065b4be457a5f3227172a87695c4f82d205d2b7a48e82b4951dd1","result":{"confidence_high":54.926029,"confidence_low":50.717307,"sample_count":2158,"score":52.826691,"score_display":"52.8","source_stated_rank":null},"run_fingerprint":"c89e793485536a1225ce4876d3684801650fc47637610d57851f75fb7d6188a5","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0c832add770c3c8e49e81fc612076af6c83611da49294cbfd78fb589cc345784","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · hle"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-medium","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"72f209a1039f0ec06d060d898c93602e4ed61dce938c2225a05ec8f3221aa850","result":{"confidence_high":53.40221,"confidence_low":49.188175,"sample_count":2158,"score":51.297498,"score_display":"51.3","source_stated_rank":null},"run_fingerprint":"5cfa1319909bd824c667e0345b3c67c6699e5d6ea329e17c0719004a58afd24a","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"36b01e8b3b79901b486719672973e0d744c208ac7903c2e530b244c8e6543604","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · hle"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-low","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Low Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"dca3faee72682e2f6a4414535c311f35ca4bc88b3774d731a71132930d3f3733","result":{"confidence_high":45.520953,"confidence_low":41.342099,"sample_count":2158,"score":43.419833,"score_display":"43.4","source_stated_rank":null},"run_fingerprint":"9999eb32e0e196db9b5f13f7af5aa63ac0df4b8d1dcedb6200bc33a0597e1e8a","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T23:44:04.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T23:44:04.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fe0968b319203300d84a7eb909df44a5064bf9ee75a6bf688410687ab27694e7","metric_details":{"best_score_across_scorers":"0.599","model_release_date":"2026-07-24","stderr":1.550610974549837},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Zftz2yk4X5NF28JQzjN3hZ","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"fbb17761ac6cb9182bdfa96233250087a44472d8d76f545b1bcbafe7cf828e8c","result":{"confidence_high":62.93919751011768,"confidence_low":56.86080248988232,"sample_count":1000,"score":59.9,"score_display":"59.9","source_stated_rank":16},"run_fingerprint":"c5273ef45fda1dd84c276930c7123933ed75530c549ff655c66d94445045f5d0","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":62,"at":"2026-08-03T16:48:38.459567Z","basis":"evaluation_at","evaluated_at":"2026-08-03T16:48:38.459567Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"35b9a6444d11c8cf965eebae08381a29abea3d4efeda94a14c044e3efa5809fa","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-opus-5-default","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"03c5b80bc12076bde3692377327143f9fb10b85b0885c2c77299a5a3d9817d0a","result":{"confidence_high":66.45212593561395,"confidence_low":60.14720066371265,"sample_count":null,"score":63.2996632996633,"score_display":"63.3","source_stated_rank":8},"run_fingerprint":"f15117394cdd2c1428ebd9a0a159b2b2d40b918113456dbd387634017c02cd0f","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cc95cb3d235fb91b5f62fe9c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f938f35e5896bb36bd60c64094f8b57ff1bc61c69baa8d8e4c2e4068a19e01fa","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.095481,"date_is_proxy":true,"latency_seconds":39.588,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.244},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":92.424,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":null,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5","zero_shot_accuracy":94.444},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"94383db983b1e909d9574bd3212494582de16e77a28a39cf4112daee8da326dc","result":{"confidence_high":95.87223999999999,"confidence_low":90.99576,"sample_count":396,"score":93.434,"score_display":"93.43","source_stated_rank":7},"run_fingerprint":"e585c7a2310dce1ccb910deae4f6e7933539bdf942edca7be781285712bfa94a","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_10853c414d0b2e6927badce7","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3db40771faa45f1b759a41b635782d6ee85d5b9977fb95128cbf7165dc0c22d2","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · gpqa"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Max Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"556ee065548106991f9cbc7331b3bb65bd34889e27522da8dc7e696396f17eb7","result":{"confidence_high":95.971244,"confidence_low":88.847742,"sample_count":198,"score":93.232323,"score_display":"93.2","source_stated_rank":null},"run_fingerprint":"6bae61fd8bad999982804a40184ffa053642e0ed3c75848392ce3c14c7abb522","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"35de9f4d0d8d87bedd535c5ca6371d7f4829d5a7a44551cdcfcce92530230165","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · gpqa"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-xhigh","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ff7dd13fbafd3212d005616d6703226dae9ebb5430d4897e772a7171b57870fa","result":{"confidence_high":96.349635,"confidence_low":89.460228,"sample_count":198,"score":93.737374,"score_display":"93.7","source_stated_rank":null},"run_fingerprint":"c173ff3980e840986aae8b5643ed1eb917abdf963e5339395d72360d4918fa7f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5975957b00cbe499f7082a7e9787847179155b78c6f9d423d72fb687b9f0c879","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · gpqa"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-high","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, High Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"3edabe9fd7742277953c6b6e76fcb4f9c9b9e5fb70789539e6a481520dbeff4f","result":{"confidence_high":96.349635,"confidence_low":89.460228,"sample_count":198,"score":93.737374,"score_display":"93.7","source_stated_rank":null},"run_fingerprint":"6477b7b623602e7851225f2e3a7422316719bf8564e6a5dce5b120e6ed8f8161","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3ccfe3e5dfc9d5846c9a4b2605bab90e7008900dcc4928bcd157c03563fb5650","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · gpqa"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-medium","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9c69f48c4241891cc78ce7e810a1302c7b4418510c131a8fe863e4ee8ab1d518","result":{"confidence_high":94.965014,"confidence_low":87.277696,"sample_count":198,"score":91.919192,"score_display":"91.9","source_stated_rank":null},"run_fingerprint":"7791f3a39db84499f32a0dac9aefccacbd3978a1383ce7e72ea8adbf65cca1f7","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"077ce19fb9ebdad207b0cc839e449846959b9ab84a65c4f70c9a393c2f0d747c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · gpqa"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-low","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Low Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6d2066914de713da31329f90606d6300d8084e5e082809f65354dadfe290af0c","result":{"confidence_high":92.547092,"confidence_low":83.750362,"sample_count":198,"score":88.888889,"score_display":"88.9","source_stated_rank":null},"run_fingerprint":"d088b749a339d86c9b418b499a70a1804999612c45218df7da5b774a2f7cfabe","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":72,"at":"2026-07-24T18:40:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-24T18:40:39.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ffe9708f1860c68bc9a07a9a5f5a9e6d95e0fcdf83596f51e175a1737d4ab0be","metric_details":{"best_score_across_scorers":"0.9387626262626263","model_release_date":"2026-07-24","stderr":1.4782015056089346},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"D3mtasTumfvjR4fjmnHWRy","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"31adac5177cfacedbf04ff98e1c3cb1b280e40a73c58e7d1dd8bba3669c9c468","result":{"confidence_high":96.77353757725614,"confidence_low":90.97898767526912,"sample_count":null,"score":93.87626262626263,"score_display":"93.9","source_stated_rank":14},"run_fingerprint":"4c5313638abcc06c7840d3b42d0527fa7c88a72043529977df283a6050774370","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:21:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:06.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"85c02d161f456b74ff03562808d1c5c3f9ee68ff43ec9aa98a7e2ae0b84575d7","metric_details":{"best_score_across_scorers":"0.8787878787878788","model_release_date":"2026-07-24","stderr":2.325315795194205},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"WfaJS4L7Xdz5ZBXgJMbRGb","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWfaJS4L7Xdz5ZBXgJMbRGb.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/WfaJS4L7Xdz5ZBXgJMbRGb.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"3db7967125cf16a04db099b6da24de8eb627d39962a41c1e997ab3f96dc882ff","result":{"confidence_high":92.43640683736852,"confidence_low":83.32116892020723,"sample_count":null,"score":87.87878787878788,"score_display":"87.9","source_stated_rank":63},"run_fingerprint":"7757b03c3bbed93df3a0a80282aab5cf5513056377b25af95d2820e8e2b36e8d","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:21:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:07.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a62b0c7c74660ab4b628cdfedca4e26894c96e56e7cbacd6d4b7e89bd1b846b9","metric_details":{"best_score_across_scorers":"0.9292929292929293","model_release_date":"2026-07-24","stderr":1.8263105420199484},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mLmUnBuX6DYKtb6x46GteM","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmLmUnBuX6DYKtb6x46GteM.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mLmUnBuX6DYKtb6x46GteM.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"891b15a60b596df2e3cd4ee26e9ddebb4dda985b1666626bebb067679c5e8f1f","result":{"confidence_high":96.50886159165202,"confidence_low":89.34972426693383,"sample_count":null,"score":92.92929292929293,"score_display":"92.9","source_stated_rank":21},"run_fingerprint":"3915fdcfc192f371eedade2392a6ad797c25c45ae6bac5d064fc145f61c10d61","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ec8010721ef20331cd53dee0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":69,"at":"2026-07-27T21:54:55.504714Z","basis":"evaluation_at","evaluated_at":"2026-07-27T21:54:55.504714Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d82e24b3f9c8ee894a1120c9271550ecc5d8258626b767690faafd8ec9304136","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-opus-5-default","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c55aef84329ed859f5484510ec0be3304cd62e6a4b00c141adb357f394fdb166","result":{"confidence_high":94.09634105718209,"confidence_low":85.7016387407977,"sample_count":null,"score":89.8989898989899,"score_display":"89.9","source_stated_rank":7},"run_fingerprint":"e32366b36841aefde009bc5453cdd1c4c89936568718e0dec2b580b40017a4c2","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7113f90e84ccb655ea073439","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":74,"at":"2026-07-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-23","status":"fresh"},"measurement_id":"e53d2599b290f50680aec10b68e8743e09893c496abadf91fe1030ac254715c4","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.027843,"date_is_proxy":false,"latency_seconds":9.6,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-opus-5","source_locator":"Results · Overall accuracy","stderr":0.276},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"65dd939d4410aacd3a40647c3dabe35d09e7b5a0bdec81028d3005e8d136789a","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7243434f5a7e55a2297631732b3f4d04a03f308bb5a452dd387c6c5e5227451c","result":{"confidence_high":92.13195999999999,"confidence_low":91.05004,"sample_count":null,"score":91.591,"score_display":"91.6","source_stated_rank":2},"run_fingerprint":"54e4b2e52041da32d071ed9dc6ff908fa378329d72826ade9182b0a88b03b313","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":72,"at":"2026-07-24T18:40:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-24T18:40:39.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f3ab900f2b6880555cff5de9597da15f58d89048b1bc034be33d3d004cf0885c","metric_details":{"best_score_across_scorers":"0.9888888888888889","model_release_date":"2026-07-24","stderr":1.1111111111111112},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"KxvHAT9SV2dTDn7jBnTk6f","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a9d9b4fc68968f9abe8cf470be2c363f3e52f5d85ac66dac5e30fc8c98d92115","result":{"confidence_high":100.0,"confidence_low":96.71111111111111,"sample_count":45,"score":98.88888888888889,"score_display":"98.9","source_stated_rank":17},"run_fingerprint":"6cf730ac140651cef08e9b82e059c775249aa72cd47a405e011bdddf9a89a0da","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:21:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:20.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cdbb0128414130a81113ab1734f0e5ac951c1ae56f5530afd4b79304f25fea9e","metric_details":{"best_score_across_scorers":"0.9333333333333333","model_release_date":"2026-07-24","stderr":3.760507165451774},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Z5FAHm8aqmk8NUUm7tojng","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ5FAHm8aqmk8NUUm7tojng.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Z5FAHm8aqmk8NUUm7tojng.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7e79ea882e8f162618fcba6407ed1a889d727ebcc685f67e6f2753a599d72e39","result":{"confidence_high":100.0,"confidence_low":85.96273928904785,"sample_count":45,"score":93.33333333333333,"score_display":"93.3","source_stated_rank":53},"run_fingerprint":"b2b491219b31741573e7fd1d93182d8f8e9e7456899c2a0d05bab9e1d3ad212e","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:21:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:15.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"456783e7fd11151b078dae3126409a34f405e3f5e9ed3ccbebfce9ee5de6071e","metric_details":{"best_score_across_scorers":"0.9777777777777777","model_release_date":"2026-07-24","stderr":2.2222222222222223},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gsBidnvEcmCfLnn5sxePtk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgsBidnvEcmCfLnn5sxePtk.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/gsBidnvEcmCfLnn5sxePtk.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"dcc9fed77821617c7027a68a6ed255c9f89887b8a37401fcf80e624f51570602","result":{"confidence_high":100.0,"confidence_low":93.42222222222222,"sample_count":45,"score":97.77777777777777,"score_display":"97.8","source_stated_rank":25},"run_fingerprint":"60c49d7abc667f85895175384cfc932c59f7cf2be3b5bec879ef420502499e3a","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b903824dfce1d32be0ee295","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":72,"at":"2026-07-24T19:54:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-24T19:54:20.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6123005ed5ad66ee49c8241c370709433a948587c7ba056edd677daf349fde79","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.856140350877193","leaderboard_private_problem_count":285,"model_release_date":"2026-07-24","public_example_count":10,"stderr":2.082489457534394},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"ff7GeS7rdperjxSBZ7MHZY","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fff7GeS7rdperjxSBZ7MHZY.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ff7GeS7rdperjxSBZ7MHZY.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 5 (max)","thinking":null,"upstream_model_id":"claude-opus-5_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"e8950252a5c51330728bc6367a249c0cc704fac117b382f6fe9b1d068b0f8aee","result":{"confidence_high":89.69571442448671,"confidence_low":81.5323557509519,"sample_count":285,"score":85.6140350877193,"score_display":"85.6","source_stated_rank":11},"run_fingerprint":"3d90cd1e8578d3fe770f3e13f53f2efdb145cd642f3375c2c614517734e9da5b","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5ab372caf4209c5902a46f7720f358aa3136e47768049307ce685dfcb1e8036e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":2.06,"model_release_date":"2026-07-24T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-opus-5"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-opus-5","model_type":"CoT","upstream_model_id":"anthropic-claude-opus-5-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e4375a17736c8f613f4c9ef2a1a2c24cc839e2fc69691eeec35e9a9796c2841b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.42,"score_display":"90","source_stated_rank":13},"run_fingerprint":"2668d4e003509c10d081c6433ea8531872ae51320cbb53337b0408dfe440e331","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6efdf7182d08ba7fe5b2bb95c7e2a47265f757e6ff295f2de7fa90564f85a5d4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.45,"model_release_date":"2026-07-24T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-opus-5"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-opus-5","model_type":"CoT","upstream_model_id":"anthropic-claude-opus-5-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"65c1a79564641a289b0ebac8f67b3b2ddb05605e04457d5304cfd59c154cd7a2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.33,"score_display":"88","source_stated_rank":22},"run_fingerprint":"a4cc65188fe117153f80076e7ef790fc3fec137f88e8fa476b1d9d3966dea50e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"395f7f886dbc0c88b46b7a322c9987d5543fff5bf7086b17afee2543d5ab419c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Max)","source_effort":null,"source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"8ed0b25565d81b65a71b28f440ce475e32e495e6934995d3d743bf5b4e90eb4f","result":{"confidence_high":93.047422,"confidence_low":86.939033,"sample_count":360,"score":90.42,"score_display":"90.4","source_stated_rank":12},"run_fingerprint":"0ee3205f4592f48aa3fd596ea9b1a7b257ed5bc53fa7df6f1e0c9411480962a0","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c46e9f7bfc570d4731221a1fb8d82d0ed491c34ef3749d39dc05dd39c3652e0d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (High)","source_effort":null,"source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b802622abf31ff6e97212a55a8f4d5661ec508e38b03816e84788183fa90989b","result":{"confidence_high":91.24908,"confidence_low":84.60151,"sample_count":360,"score":88.33,"score_display":"88.3","source_stated_rank":21},"run_fingerprint":"cb2142896dfe8740232ba6b4586a27953e43655bcf3b66c8fed703ab6611d5d8","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"caa5f0885ccc3ebca568c2e1ab65ceafac4e1202e68e7494b94594ba8d9d2082","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (High)","source_effort":null,"source_model_version":"claude-opus-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b5f22d18362bbec091833d47d2fe78dae74f033db59bf3554d99275b0299a5d6","result":{"confidence_high":91.24908,"confidence_low":84.60151,"sample_count":360,"score":88.33,"score_display":"88.3","source_stated_rank":21},"run_fingerprint":"389b3e009c282c2e8c509ec3483d9bec4b48a07dbd30a5425f1533c42e849cc0","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":72,"at":"2026-07-24T19:54:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-24T19:54:20.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1d32fe0686373b3681a0cd8c8aabfd25fa1ee4cfc46a01a233eab4d069598b25","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.7317073170731707","leaderboard_private_problem_count":41,"model_release_date":"2026-07-24","public_example_count":2,"stderr":7.005564203095157},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"ji3kMn49Jcr9t4whxcpjj4","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fji3kMn49Jcr9t4whxcpjj4.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ji3kMn49Jcr9t4whxcpjj4.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 5 (max)","thinking":null,"upstream_model_id":"claude-opus-5_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"e3cf9103efd2521f7be4eacfc07310d73b84b17da7d89c4f1b7663c6396f3cf7","result":{"confidence_high":86.90163754538358,"confidence_low":59.43982586925057,"sample_count":41,"score":73.17073170731707,"score_display":"73.2","source_stated_rank":15},"run_fingerprint":"9ff764decaccbce7522784eb603b900ce473ea2c3743308cfbbbc818f02195ab","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-07-23","status":"fresh"},"measurement_id":"4f552b7a92105f3ed5e523694439e99bf1b031a1d8798ac28b3ba0f625c7b92b","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.069014,"date_is_proxy":false,"latency_seconds":30.028,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-opus-5","source_locator":"Results · Overall accuracy","stderr":0.725},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"bae5570e4d9b683e073ff86643ab4b9f8f9ce3387a2bdcccecd9c1100b98da97","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c6f814bf3c7305a3fecd9f959c4444b399ace8cf516b9f6d4c03f7f53d78fc43","result":{"confidence_high":91.305,"confidence_low":88.463,"sample_count":null,"score":89.884,"score_display":"89.9","source_stated_rank":2},"run_fingerprint":"02add8891da72156489c419b3fd546239416e6912a620bfdb4e380f631d201d4","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eecd46743d753886d8ecd1a303588bc1f43e5abc0ee62282d6891859f6f33a15","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Max Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4f7ce0b1c1211ef05ec237455d31ee812c3c39a09587f00671faec780bf2a4b5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.739884,"score_display":"84.7","source_stated_rank":null},"run_fingerprint":"a402425aeac3b81c1189adcf06325bd6b14028448ce805da2b3247d1fa537a90","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ab7caac96f18c65e122cd339b233a6aeb11cd66a4060d0414d57497e6e86ee00","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-xhigh","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d53bdee2deacc904cb64a4f0809590ab11b356e5e80d2f3999ee94741a902c6f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.988439,"score_display":"84.0","source_stated_rank":null},"run_fingerprint":"1d446f48a89a7e9bdac9cacdd01c99b9f04f133d339ac524e9733cb257b41e80","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0d5fec307d4bc39efd1ba948d585f96669ebaf01e378227a4b2d6035141527e9","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-high","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, High Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2da8eb4d57843f9973c322c3e0b3e760b32d519d4de775b3955019e77bc626bc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.427746,"score_display":"82.4","source_stated_rank":null},"run_fingerprint":"49d17ebf4885e078761a84beb877e1a89964d02e8a40fad9c92838bed3bd4d60","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bbcf98c796ea33f0f97ccce1669f185b1f00cf284343c64b9e534b84c0914b98","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-medium","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"78ae80f22cc14337c82e54431e4e55c6507f9e41e6ccffc3f46da91bf01547c0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.618497,"score_display":"81.6","source_stated_rank":null},"run_fingerprint":"af190e8496f70ba67a77362c9f7a0d99ebcb110a7bc8e430fad4d8578e65dfe2","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"58c4e59b6c7ef19fd1bf1fec00bfaa49db14c28e8e0201da2467465602ad488f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · mmmuPro"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-low","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Low Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f9517c952974e1b10afdb43230cc9028f86fb8b13038d2f72cc3b3efb7a52695","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.82659,"score_display":"79.8","source_stated_rank":null},"run_fingerprint":"cb75b226c903541c992e568d8a921e10f2f586ab55637ab25a561507abd23851","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b1ef81b096787ad4a1e0dffa9878dfeb24f298cad152ded2e9dba29a58f18bec","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"09b05cc465e646d9e64a872d732862c4548a4d4ca19620c3b74478c312069c54","result":{"confidence_high":40.564344,"confidence_low":19.862553,"sample_count":71,"score":29.1428571428571,"score_display":"29.1","source_stated_rank":19},"run_fingerprint":"d3c7d94f1828601b77ffa7050d2b6c751d61f4de93f8ecdf8f1d6d8c61629f54","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"12f0064c19832cbde186edea61349b03adf398f955eea5145a085041e7dbca43","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)","source_effort":null,"source_model_version":"claude-opus-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"1a769ba0e537ef167755690c4c0b3874f78b0b719f5c095ecf7aad929116ab13","result":{"confidence_high":39.063098,"confidence_low":18.653313,"sample_count":71,"score":27.7142857142857,"score_display":"27.7","source_stated_rank":27},"run_fingerprint":"a69ed2e18624bc6a2c6491a18774c21cb9a710121aef8f016dc8af3eadc3ffed","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6e9879e8c9cd239a9fd9d2fe18f09d28816ca7b8fd9d17a8ad4cc9782f070f4d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, High Effort)","source_effort":null,"source_model_version":"claude-opus-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b59fed4ba4e0842b0513d521143ebd3c840aee869ac4e20338f8f877d6e65c0e","result":{"confidence_high":39.665008,"confidence_low":19.135598,"sample_count":71,"score":28.2857142857143,"score_display":"28.3","source_stated_rank":25},"run_fingerprint":"b4357dda6b224807c5bf14f35637657a186be1b58851aa7ba336821332d41b6c","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6fd2c8e5fe131790ea327553490e29db7782189ff6b13bb15fa50decd1856a2c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)","source_effort":null,"source_model_version":"claude-opus-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"52b31b2d57e54f39f9751bddc9b4e04b8988cd10cfb04a028fea2e08315618e9","result":{"confidence_high":38.156623,"confidence_low":17.933497,"sample_count":71,"score":26.8571428571429,"score_display":"26.9","source_stated_rank":33},"run_fingerprint":"b1792c2b6ecc154639fc151e2e9832ac8f85ea992fa5e19bc058eb4ac753c323","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"39ec8b9b88222ddc4ddb0052adf13078bf1b460ed855cce63f54100d018560b2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, Low Effort)","source_effort":null,"source_model_version":"claude-opus-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8174d7067bcbe20230a7880cca7fdc80f927f746cacea6f48baf16d9757085cf","result":{"confidence_high":34.17546,"confidence_low":14.867394,"sample_count":71,"score":23.1428571428571,"score_display":"23.1","source_stated_rank":45},"run_fingerprint":"3a37ef73f25b2b55823861f62d89251602f4a811b7bc240b39dc7d8ae93da3b4","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1bc7507cc5333de11b7d5b8db6e06a01fbdb8be732146be8349ffcb0e13b336f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · critpt"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Max Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"100eb23ff6be8c58ad8a7d861932b0ec105c41f75fbf32a2641529ae49d4b30f","result":{"confidence_high":40.64945,"confidence_low":19.806444,"sample_count":70,"score":29.142857,"score_display":"29.1","source_stated_rank":null},"run_fingerprint":"1627a2d2d51b562839541a769784360411997be3e77f2325de011509bdacd09e","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9dbc77ea4c49177b5ae18f11bda1cadfd64d62bd55dbc5f1a364a1914d962170","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · critpt"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-xhigh","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5e7460347e3830099244ba683d358c9e1dca8da9666593640809403a24c8a73d","result":{"confidence_high":39.148332,"confidence_low":18.599063,"sample_count":70,"score":27.714286,"score_display":"27.7","source_stated_rank":null},"run_fingerprint":"0d451dd850a1413bda5e6b258317c72fa326f7f5c7ec62a7a8877568e249241c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"51f728470b55bd6070fcf47ef6d6dce87cd0d814d4b121303d3f41404d24a64e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · critpt"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-high","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, High Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"52f05a327cb80d585ed91adda3a512b95e1df5fab4c024a687ae5517f226ffef","result":{"confidence_high":39.750198,"confidence_low":19.080596,"sample_count":70,"score":28.285714,"score_display":"28.3","source_stated_rank":null},"run_fingerprint":"33ed5545d30d9c1d4244471228294cb704c9e520a3ee63f297314103df6b3322","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"876ce7f6007b87ecb5c9ab6360deeda3483c7401fd5b2d2c319f9fc4d8afc363","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · critpt"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-medium","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b0733a2f332f2e63d5b155ae114b80ddabe680743fd0405e9fda034d0407dea9","result":{"confidence_high":38.241899,"confidence_low":17.880395,"sample_count":70,"score":26.857143,"score_display":"26.9","source_stated_rank":null},"run_fingerprint":"b2d1d97328beba69e753e70fed1e98de891e8f805045cd31a8c1366dabb69f3b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d4c5ce3f39bf144838f27e196a1bf23c370fa58691fd7ccc289cf44dfc4ccaaa","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · critpt"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-low","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Low Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"db34da1647dafc614008cd4de4d00b9d40c08f6ce8c00755476b2e99ab7d2424","result":{"confidence_high":34.260621,"confidence_low":14.819572,"sample_count":70,"score":23.142857,"score_display":"23.1","source_stated_rank":null},"run_fingerprint":"56b5beefe0e2911c993907dedf09762289b569113147ade43a4c8b11ba94691f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6219d2c14a2f1b8a1199718029442e1c438d7684929334c2c79cfa682418fe9f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-5_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"45dbeb835a69df9d86f682a1b1ef7127e521f78f7e45acad94d8565d9d92bf83","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.6,"score_display":"80.6","source_stated_rank":2},"run_fingerprint":"2da7cc3376b3340854f4c44538aad58876f928a42aa4c9d55c5ae568cebb2cfe","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e69018c538773cbfb94889c7","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":72,"at":"2026-07-24T18:40:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-24T18:40:39.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4ec38cced2a0d1a8ae8d7ffab79f623185b72b7e97f4028a0cc3104c8d894221","metric_details":{"best_score_across_scorers":"0.42","model_release_date":"2026-07-24","stderr":4.9604496374885825},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BiaKGj9CUTKhhNwW9SM4qt","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"98507aa5eab8d637bc9e42f1c4137957b3b8ba2b65c25b12e7159d997799b718","result":{"confidence_high":51.72248128947762,"confidence_low":32.27751871052238,"sample_count":100,"score":42.0,"score_display":"42.0","source_stated_rank":21},"run_fingerprint":"0ac0816679a0d9872a03748f392fe37b20333bf62d0969fd8e3c397f69fbf1f8","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:20:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:20:31.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e8c77ecd87d8dda8b3c60a59ff575ec971fd8b05cd7a5d6b87208d7d6ba4d822","metric_details":{"best_score_across_scorers":"0.2","model_release_date":"2026-07-24","stderr":4.020151261036848},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"3dXpsUjaVEbBN72dnw4Vzq","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3dXpsUjaVEbBN72dnw4Vzq.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/3dXpsUjaVEbBN72dnw4Vzq.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8932c20446e2b7866d79b61b8c0b915d8b7c7230e0191ce03bbe5ce6b833c253","result":{"confidence_high":27.879496471632223,"confidence_low":12.120503528367777,"sample_count":100,"score":20.0,"score_display":"20.0","source_stated_rank":87},"run_fingerprint":"a6ca669f75ff7ddcf626e82cf3ef645c749eb7fb565becaa4f034b998736b718","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:20:28.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:20:28.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2b87014f289805ac955509ab2698c866b1fda68fccd28d7c4165defa7cb27061","metric_details":{"best_score_across_scorers":"0.33","model_release_date":"2026-07-24","stderr":4.725815626252608},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"UxakyoDNbLKtxnxeMsa7cU","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUxakyoDNbLKtxnxeMsa7cU.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/UxakyoDNbLKtxnxeMsa7cU.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"31a9dc1e79f58d34d79d9b7e68a408166d3c743332fd2e5f1821f246dbc7acf3","result":{"confidence_high":42.26259862745511,"confidence_low":23.737401372544888,"sample_count":100,"score":33.0,"score_display":"33.0","source_stated_rank":45},"run_fingerprint":"97b1b1c43ed7d304cbfb696eb67a86e1be6fcf7bdccd2563983d6c545710cf42","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bc4ba9896cf89df0b5500c47","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1fc58571379480df1b489cd0040708c3bdef18c6039afa9c16362a18bef25145","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (High)","source_effort":null,"source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b3ac1059018444402272d7dfd2f3ca5040faf0501c3b838bc2934dab3b7ccd96","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"157d0ef82f1235890e5b74a7b66ed31b02163074e02184a4c1aa834cdc4bdbbc","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e4ab16f94179644c08a2a49cccbff455ec392d5207db3fb152d3d2245f7f666a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Max)","source_effort":null,"source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"cef01bbdd5ee22a84ff91aa52b7325584fc1a0cbfe08ea6784de16443f1c81d2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"fe8995bf556bb1e7c5007986c72e4d2d80e445b2ee68e6ae2d8f8d86ddbe867a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8840cd5b5b582947dc01486ed59b8cd5f138e6386f3bd17bb13955c987f1ea9f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (High)","source_effort":null,"source_model_version":"claude-opus-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6f0b15adfaf685506a565f188e57ef6ec81ae3d7f3878daa9bd49abb1dcb31d5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"0535636aa91e7ac37c513023d925f070544dc782e7f69f429018d69821d2bf18","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":74,"at":"2026-07-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-23","status":"fresh"},"measurement_id":"2af688dbe169b718dd64eed619f9381ff6747ee4528abbb24fd413bc71396ba8","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run; vendor scores using a different agent or tool setup remain separate supplemental runs.","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","cost_per_test_usd":1.290782,"date_is_proxy":false,"latency_seconds":576.988,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-opus-5","source_locator":"Results · Overall accuracy","stderr":0.764},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":null,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"65dd939d4410aacd3a40647c3dabe35d09e7b5a0bdec81028d3005e8d136789a","temperature":null,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"1e55ad01ee6a328cc67243f326fac51ea5c030be5cb47d4bebca6ac453fa465f","result":{"confidence_high":98.49744,"confidence_low":95.50256,"sample_count":500,"score":97.0,"score_display":"97.0","source_stated_rank":1},"run_fingerprint":"7a7aec7c73fdcc2e88b3168433f06531a7a8296aaf095b40c488405837bc13e1","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_37b9c147fcce0c9004cd857f","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ed49ef6cb40ad079db5a0835","split_or_window":"verified-500","unit":"percent","version":"Anthropic 500-task evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"e6dc434723dc6f372c9b6b4e5d2d52ded60edb4f8f02902409e770e0c9771c42","metric_details":{"comparison_warning":"Vendor run over 500 tasks; do not equate it with Epoch runs over 484 tasks.","comparison_warning_code":"swe-vendor-500-vs-epoch-484","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.2, p.153"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"swe-vendor-500-vs-epoch-484","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; five-trial average","question_count":500,"reasoning_effort":"max","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"not reported"},"run_count":5,"scaffold":"unspecified agentic harness","tools_allowed":"not reported"},"protocol_fingerprint":"5b00a05592ee416553a42a6423ee1e8bd6262a1f79c843304dd591526b353420","result":{"confidence_high":97.395941,"confidence_low":93.902594,"sample_count":500,"score":96.0,"score_display":"96.0","source_stated_rank":null},"run_fingerprint":"37384fe11c13a520278be8f249dade2e2dc6a13fb4106847fd8986db8c4081f5","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.2, p.153","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_54c2373720fce61d7d0d3fab","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"534ea6f8fd949313d734fc37f3effabc06181a7b94be37a4f1c72397e7625d7c","metric_details":{"model_release_date":"2026-07-24","pass_at_5":74.77477477477478,"potential_contamination":true,"sem":1.35435102507864},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"contamination-risk","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"Opus 5 [high]__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"84f6c5379270272ad0d9301ed4bffb2a3624efd6c9a660db3514983cf7d208a7","result":{"confidence_high":66.07795143257756,"confidence_low":60.768895414269295,"sample_count":null,"score":63.42342342342343,"score_display":"63.42","source_stated_rank":3},"run_fingerprint":"b36b200c7b072353d662d1643535ea29a31a49d221f9404dc351414c716e219a","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"contamination-risk","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"ecac36582e53fab2d8740ec8a0e7393c1d7c6f47e82865750f540e14d66a9c15","metric_details":{"ci_attempted":444,"ci_half_points":3.872310426371729,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":327,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":99.0427927927928,"mean_cache_tokens":14784785.963963963,"mean_cost_usd":11.837583271396396,"mean_duration_seconds":1911.819866231982,"mean_input_tokens":15025834.39864865,"mean_output_tokens":117565.6936936937,"median_agent_steps":90.5,"median_cost_usd":10.4281505,"median_duration_seconds":1801.1237265,"median_input_tokens":12130307.5,"median_output_tokens":113366.5,"median_output_tokens_to_pass":112874.0,"median_peak_context_tokens":215810.0,"n_attempted":444,"n_passed":327,"n_tasks_attempted":113,"n_tasks_passed_any":100,"pass_at_4_points":88.49557522123894,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_5_max","source_model_version":"claude-opus-5","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"ace259fd68d7198838f969ae1b26d71e836fb71ca059c1b781a51b215bf35605","result":{"confidence_high":77.52095907502039,"confidence_low":69.77633822227692,"sample_count":444,"score":73.64864864864865,"score_display":"73.6","source_stated_rank":3},"run_fingerprint":"f3f86ae4fbc2ff3e6d57e2bd3839536265a211be3cadcb7241268bd4a09778e7","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"62943f6aaf30aeb264897fb3650e60a56a82ca2c882f322672383366df8c65f1","metric_details":{"ci_attempted":447,"ci_half_points":3.0609818216384643,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":327,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":88.7248322147651,"mean_cache_tokens":11094990.232662192,"mean_cost_usd":9.07220232326622,"mean_duration_seconds":1559.5597273221476,"mean_input_tokens":11292729.463087248,"mean_output_tokens":91672.01565995526,"median_agent_steps":80.0,"median_cost_usd":7.928816750000001,"median_duration_seconds":1448.063854,"median_input_tokens":8932820.0,"median_output_tokens":87322.0,"median_output_tokens_to_pass":87949.0,"median_peak_context_tokens":173749.0,"n_attempted":447,"n_passed":327,"n_tasks_attempted":113,"n_tasks_passed_any":97,"pass_at_4_points":85.84070796460178,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_5_xhigh","source_model_version":"claude-opus-5","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"564b6995570ddaada93cc698defc2e3014b7f8f099599f9b015a59d470e39e49","result":{"confidence_high":76.21534423774585,"confidence_low":70.09338059446893,"sample_count":447,"score":73.15436241610739,"score_display":"73.2","source_stated_rank":6},"run_fingerprint":"a804777608e0a52228486a06aa23f39efb8f02b28c4dc3b1926f6a81bb8a9de6","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"0bc063e11d43f085000348bfc7b7a9f28d593dfe77022634bfc68e17914d6730","metric_details":{"ci_attempted":449,"ci_half_points":1.9454321571601496,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":327,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":72.92204899777283,"mean_cache_tokens":7085027.516703786,"mean_cost_usd":6.076084489977728,"mean_duration_seconds":1163.8627881514476,"mean_input_tokens":7233879.0824053455,"mean_output_tokens":64207.43429844098,"median_agent_steps":64.0,"median_cost_usd":4.96882375,"median_duration_seconds":1006.103652,"median_input_tokens":5212804.0,"median_output_tokens":59856.0,"median_output_tokens_to_pass":60897.0,"median_peak_context_tokens":126517.0,"n_attempted":449,"n_passed":327,"n_tasks_attempted":113,"n_tasks_passed_any":99,"pass_at_4_points":87.61061946902655,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_5_high","source_model_version":"claude-opus-5","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"1c3b703def7720e01c3231b948c26783875ce831a6a432a9e75be843093909a6","result":{"confidence_high":74.77393995226038,"confidence_low":70.88307563794007,"sample_count":449,"score":72.82850779510022,"score_display":"72.8","source_stated_rank":7},"run_fingerprint":"45fe5dad5e6d4840bfaedcfb212c104667bcd16357e65db18d2c04c305492a75","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"cfd082242e5480b122592fb7d268cbdc5ba8365b45f2cf39c3c63155a51bb128","metric_details":{"ci_attempted":447,"ci_half_points":1.1725586658930944,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":308,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":52.29530201342282,"mean_cache_tokens":3479530.087248322,"mean_cost_usd":3.28978764541387,"mean_duration_seconds":759.2254874630873,"mean_input_tokens":3579815.3601789707,"mean_output_tokens":36981.530201342284,"median_agent_steps":43.0,"median_cost_usd":2.4819885000000004,"median_duration_seconds":587.101605,"median_input_tokens":2211638.0,"median_output_tokens":33436.0,"median_output_tokens_to_pass":33678.0,"median_peak_context_tokens":78690.0,"n_attempted":447,"n_passed":308,"n_tasks_attempted":113,"n_tasks_passed_any":101,"pass_at_4_points":89.38053097345133,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_5_medium","source_model_version":"claude-opus-5","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"37167dc5c838b8d5695580deb3e717f040c659508519500b2a0ba9d01d55d545","result":{"confidence_high":70.07636179788415,"confidence_low":67.73124446609795,"sample_count":447,"score":68.90380313199105,"score_display":"68.9","source_stated_rank":17},"run_fingerprint":"7505f04af58853e0a694e98d5a6c2d80a538bc50f81895eaf435e47c35a80dd4","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"540051883eb906e26344e7e70f86f18c9ceca329a505e024bd3bf2d367911955","metric_details":{"ci_attempted":449,"ci_half_points":2.3306879466842725,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":261,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":35.641425389755014,"mean_cache_tokens":1497598.9933184856,"mean_cost_usd":1.6626223919821828,"mean_duration_seconds":468.87542417149217,"mean_input_tokens":1564287.6570155902,"mean_output_tokens":19884.3140311804,"median_agent_steps":29.0,"median_cost_usd":1.2162672499999996,"median_duration_seconds":333.286568,"median_input_tokens":872162.0,"median_output_tokens":17613.0,"median_output_tokens_to_pass":17502.0,"median_peak_context_tokens":46865.0,"n_attempted":449,"n_passed":261,"n_tasks_attempted":113,"n_tasks_passed_any":96,"pass_at_4_points":84.95575221238938,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_5_low","source_model_version":"claude-opus-5","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"0b231141683bd26fda5a664cea71afc44e036169cd9894e52894e6fec4bd7cad","result":{"confidence_high":60.45986389323216,"confidence_low":55.79848799986361,"sample_count":449,"score":58.12917594654788,"score_display":"58.1","source_stated_rank":36},"run_fingerprint":"56582530f20ce7d313c3d7faf05d06cc46cea335f3206d2a67afb8e20d1209d6","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"88309d2f645fa9358326a6b01be5d6662db6922dadb3d22e19b711d5741fff51","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"99.0427927927928","mean_cost_usd":"11.837583271396396","mean_output_tokens":"117565.6936936937","model_release_date":"2026-07-24","notes":null,"pass_4":"0.8849557522123894","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-5 (max)","source_effort":"max","source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"ac4b9332df3ecd0009dabbb54fbebdbefb0cf4b817508684294b9968b8c04ee2","result":{"confidence_high":77.52095907502037,"confidence_low":69.77633822227692,"sample_count":113,"score":73.64864864864865,"score_display":"73.6","source_stated_rank":3},"run_fingerprint":"fc99abb18b4cf17da4c426ea03cc76b2fcbaf227909d205b050be6789d375433","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6d3ec24539db7a9c328d04a00f83c1a85735eeef188be4fdd8505e1fac9d1145","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"88.7248322147651","mean_cost_usd":"9.07220232326622","mean_output_tokens":"91672.01565995526","model_release_date":"2026-07-24","notes":null,"pass_4":"0.8584070796460178","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-5 (xhigh)","source_effort":"xhigh","source_model_version":"claude-opus-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"ce293f30092fea94ff725a86934e78eba7c475bbcc4f0d7db8a0c7584c8988db","result":{"confidence_high":76.21534423774585,"confidence_low":70.09338059446893,"sample_count":113,"score":73.15436241610739,"score_display":"73.2","source_stated_rank":6},"run_fingerprint":"8b34b9c1974171eee6898ca15ecfe685f8236cd6826468e510bf1c431a3df6e7","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"237676c970966fa4183cd9e615665f0ee760e7cdbd42312b38d338e954979ae6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"72.92204899777283","mean_cost_usd":"6.076084489977728","mean_output_tokens":"64207.43429844098","model_release_date":"2026-07-24","notes":null,"pass_4":"0.8761061946902655","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-5 (high)","source_effort":"high","source_model_version":"claude-opus-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"8f86bc090d7321d132658c05002ab85a918cfa1ec21783574ad9207a0339fa58","result":{"confidence_high":74.77393995226038,"confidence_low":70.88307563794007,"sample_count":113,"score":72.82850779510022,"score_display":"72.8","source_stated_rank":7},"run_fingerprint":"e0fc5d69cefbcb3906c162a1696128b6b54a2c29d39b6698e341b9ede2b096d1","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e1b3d1106770357fc539fb9b6cf81a31cbc5a6ceaaae173853a1ce33c06732aa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"52.29530201342282","mean_cost_usd":"3.28978764541387","mean_output_tokens":"36981.530201342284","model_release_date":"2026-07-24","notes":null,"pass_4":"0.8938053097345133","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-5 (medium)","source_effort":"medium","source_model_version":"claude-opus-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"a1c15c0d8ba3289a1256278f17499d811b51a1f814838080c47979ba73424305","result":{"confidence_high":70.07636179788415,"confidence_low":67.73124446609796,"sample_count":113,"score":68.90380313199105,"score_display":"68.9","source_stated_rank":17},"run_fingerprint":"7f015b18d14077827d6ea475124bc08535e6162fae4724e857257ebba417e40d","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bd0dc9da121fb1a1fdccd8f5b614b4cd54e715d609aea9c07aaf3b2d1e035487","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"35.641425389755014","mean_cost_usd":"1.6626223919821828","mean_output_tokens":"19884.3140311804","model_release_date":"2026-07-24","notes":null,"pass_4":"0.8495575221238939","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-5 (low)","source_effort":"low","source_model_version":"claude-opus-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"e1897f641455248af965fe0990be445eb2ace68d30488d56f50ce7e13a0168b0","result":{"confidence_high":60.45986389323215,"confidence_low":55.79848799986361,"sample_count":113,"score":58.12917594654788,"score_display":"58.1","source_stated_rank":35},"run_fingerprint":"922941532d0e1f0d2781790073d6bba6a5f2c5dcea731956024c72ec65d7c249","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_6323abbd504101af2cdaac58","split_or_window":"vendor-public-own-harness","unit":"percent","version":"SWE-bench Pro · Anthropic provider run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":73,"at":"2026-07-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:14Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-24","status":"fresh"},"measurement_id":"c65da914230e12f6a8d0671459f9f533941b1f02dfeeec89e8b5c6931b3121f5","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.2, SWE-bench Pro capability evaluation"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; default sampling; five-trial average; exact public-task revision not restated","reasoning_effort":"max","result_published_at":"2026-07-24","source_content_hash_method":"raw_bytes","source_content_sha256":"0950dae1ba6b341e4f1a009e535e0f025625efeca21bd043a9a5ae148a3f2e6b","source_published_at":"2026-07-24","tools":"software-repository shell and code-editing tools"},"run_count":5,"scaffold":"Anthropic agentic harness","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"dd2e37a9c83ee4f4ed05dbd94884fd624eea03f4248ca2dc3f6a9339a5524651","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.2,"score_display":"79.2","source_stated_rank":null},"run_fingerprint":"2166588d66a1f264c2621748d665e85bde15ab95fe533e30ec1093f8888cbe67","source":{"content_hash":"282bcbe0a60ea492278ec6a5994953c3d1cc2899decf416d00592fa801cfa94d","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:28Z","homepage_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf","id":"anthropic-opus-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.2, SWE-bench Pro capability evaluation","name":"Anthropic · Claude Opus 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/ceaf5c7ff2783855203fde8208ec311252dced5b/Claude%20Opus%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_54c2373720fce61d7d0d3fab","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bf209ee30cfb494dba488d3c812c0dd7789fcdd4583b4c0e447fc5fd2585b78b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"11.95","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"steps_per_task":"106","tokens_per_task":"85384","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5 Max","source_effort":"Max","source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"c1a956cb374edfc35a63f4b0308f963b6e05d1beb7431d21c2bf9a198008e622","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.6,"score_display":"46.6","source_stated_rank":12},"run_fingerprint":"2983f906bab0b84da5048d6ea55906052bd3f288e1bfee9456d57947a0605013","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7b4b10a05a137808f3b98184e6d244980b50b6a1218c1c85ff5072307dda665f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"11.43","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"steps_per_task":"103","tokens_per_task":"80094","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5 Extra High","source_effort":"Extra High","source_model_version":"claude-opus-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"7896d9a42e85b427f525650badfb2e0f46bc4e23753aa46547fd4d640743347a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.1,"score_display":"46.1","source_stated_rank":14},"run_fingerprint":"2fb60268b2105376be58234d0066df72485de3858c2504a1de4b332a81a4f34a","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7a536e2d7f04c78b32eff28cfaa83371ce80495a9a664badc278912819b403a5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"9.0","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"steps_per_task":"86","tokens_per_task":"61405","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5 High","source_effort":"High","source_model_version":"claude-opus-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"ea6690eabddf60b920adf004c7e8e8a93a59b3acb54268664fe97d89eb8dd9b6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.7,"score_display":"44.7","source_stated_rank":16},"run_fingerprint":"12235e4945f2a51f90315c11f11d02c829fbb8106add9757f6e8aee88c83c5e7","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a9730e2be455ee3ff6a082e2d20e11c2f305471aebdc4030f78a2693b3b22801","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"6.94","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"steps_per_task":"72","tokens_per_task":"45272","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5 Medium","source_effort":"Medium","source_model_version":"claude-opus-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"d2dcb30e2f7480872e77d056fd5b14d18938b89f7a840544073aaffaae69f5cb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.3,"score_display":"43.3","source_stated_rank":19},"run_fingerprint":"a056f2681b372e1c806aa1f2ecd90573e472cc81db8ffb220c9c858e1290a397","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"424da7426faececfa21094cdcba368b40715dc3783d3682e86075170eaf52a2f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.87","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"steps_per_task":"57","tokens_per_task":"31995","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5 Low","source_effort":"Low","source_model_version":"claude-opus-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"956c54d715d85fbbacd6a6ccf4baecd48b34f2603b3fd3882b9ac6a8b5d48a17","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.7,"score_display":"40.7","source_stated_rank":25},"run_fingerprint":"0b99c0b807e59a4e92745af2787cee00b3ca3c09d2cb5df431803258155319dc","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"25c5f133fde08a46cdbb15fb6a644c6eb0715fd669be2d98bcbec8e141d658c7","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5","source_effort":"medium","source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"e0feb047845803ef3ff53270763eb37c74610e5ae13172e2119df2cf213d49b3","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":53.38,"score_display":"53.4","source_stated_rank":3},"run_fingerprint":"d8033b9ebe48480038257526a4600d2bf67cab3349e08738505c70a0627e7cc4","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c0c9f872e4a681d6a7e8da3c1e575416b33c1d06487097a5092ef2ba01e72224","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":16.477241,"date_is_proxy":true,"latency_seconds":3402.045,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":9.964},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":null,"source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"e63f891b1983c2884e03ebd40a80de7246519ae1b2688c1f74f391992047c05b","result":{"confidence_high":100.0,"confidence_low":64.80356,"sample_count":null,"score":84.333,"score_display":"84.3","source_stated_rank":8},"run_fingerprint":"d4bd757bb290b554fd756f3a3982d28d53ab15f22deea7a9a435a625c3fd513f","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":74,"at":"2026-07-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-23","status":"fresh"},"measurement_id":"584d1b154385fc2c8319a34f2da421302759c36bbe85e87a72eeddedcf533f90","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.134205,"date_is_proxy":false,"latency_seconds":59.22,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-opus-5","source_locator":"Results · Overall accuracy","stderr":0.913},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"65dd939d4410aacd3a40647c3dabe35d09e7b5a0bdec81028d3005e8d136789a","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"57b78dff5363af7358e3e977db803f3979e0eecff7fc70e8638d2b4d9a840c98","result":{"confidence_high":90.82248,"confidence_low":87.24352,"sample_count":null,"score":89.033,"score_display":"89.0","source_stated_rank":4},"run_fingerprint":"91292ad48f7b333bc6e6d29bdbf28ea3e961420faf150ec236e93e84b84fd219","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"580229d0e14e8d1c3eddbe35f266459bb5e7ce232f4c70f4b1176deebb62b511","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"cd9e7560d85a0322d3b34a586a289f41e6311f90632278fdc833038056ffc518","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.3657407407407,"score_display":"56.4","source_stated_rank":31},"run_fingerprint":"22ba3a5b9e4ccc56dc69186dba9b0a00ebcf4f21ce788129c6ac56f9e7baffbe","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"449af5d64249f2fb476fc0069476cc7d4646eca8770393bd07e5158f8832dc2c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)","source_effort":null,"source_model_version":"claude-opus-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"59a42b86d25139f6b6a4775a4f42a73b2e737d526703a3f270ad0ce9d9bd6eb6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.976851851851805,"score_display":"55.0","source_stated_rank":44},"run_fingerprint":"e1b43a6d7ea505ac01c317feae9ccf02a83cac1531d069355a5f3ca07e67f3f2","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"161da1ab55aa8329c493bc9ab4082e49e5cad52c182d2f0f41a24c70f5be663a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, High Effort)","source_effort":null,"source_model_version":"claude-opus-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"d56e7bfb1d20ad0cf709e26a4585acb50e7c771af3a9f59c81aa2735aa1c953c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.2824074074074,"score_display":"54.3","source_stated_rank":51},"run_fingerprint":"ba2a3dd580fdc92ba64aa1a12cace194cc8e7bfdbb59cec312624f48b258192d","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6ba0170e9c25c3fdded4ba5d7bd3079947f13d2f83a39aef1a9e18ab23dacad9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)","source_effort":null,"source_model_version":"claude-opus-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e0e868a77553df77c3ed154377ad412ee7627fdf6969c92cd6b2815b7e7c4924","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.6944444444444,"score_display":"50.7","source_stated_rank":83},"run_fingerprint":"477c65d461d81c23a463c61ea7d75753d51531b7d33e430f964726a14fba6377","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e819adf7f05a0769b53c60b762ca2f8814aa8f7246b19631c1623beeb9518ed0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (Adaptive Reasoning, Low Effort)","source_effort":null,"source_model_version":"claude-opus-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"de8c0f6851b0da3acaf299d586a1847afd3b9f7b42253a218c8167d5ba6146c3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.032407407407405,"score_display":"48.0","source_stated_rank":105},"run_fingerprint":"b675bacf1d35e05d82a8752217649e8048cd6db4634538a1d4b11fe35a9523e4","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e0e4ca50037c28cda7d709213e3bf89169ecd238b27dc970d9f2ddbe8564af20","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Max Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b8efffbee035768e764e478983060b342a788a3e4e51a9bd874a140bc1b90f09","result":{"confidence_high":61.972454,"confidence_low":50.59144,"sample_count":288,"score":56.365741,"score_display":"56.4","source_stated_rank":null},"run_fingerprint":"4ed1d5838f90dbbf5e3b60d2d851629c8d330d7881681a097a5c2409dc1b7483","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ca194ee13c420866383f6b9a1fd683e1e2cced32a8ba9965220f91901ceeee8b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-xhigh","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"03b61b407ac2dd0eab763c21d7af4c0495f4f6f22d0f4e27bb868777ca133bf9","result":{"confidence_high":61.296683,"confidence_low":49.896603,"sample_count":288,"score":55.671296,"score_display":"55.7","source_stated_rank":null},"run_fingerprint":"bd762bdd5c22868974580fd5689c46845281bf23c9cf029bcaa2ffd695b77f48","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ec3329caa07148e23181b7b188678d02a461336c6c65dd83a5c694fff8325944","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-high","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, High Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"3639906b7042f4bfd384ea06ed506228bcdc751670f2040df492be9d1d08e690","result":{"confidence_high":61.071179,"confidence_low":49.665239,"sample_count":288,"score":55.439815,"score_display":"55.4","source_stated_rank":null},"run_fingerprint":"6799d22fbf73946c94a6009ec73ab5d768614168af6632684685c0bb1f518e10","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8c74f4d73c8f2e2204339b5d7494193edb1bd3c6d6e9d90b22adf2eb501d189c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-medium","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"266c84015b73ef11b7ebc9107d87f92a1faac46dd54a35f96acbd89867ade487","result":{"confidence_high":57.218833,"confidence_low":45.750816,"sample_count":288,"score":51.50463,"score_display":"51.5","source_stated_rank":null},"run_fingerprint":"79bc52faee4198c5c732522a02fa9a3d7a4857919358979aa77e56029a9ab79c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"99a111bbb9f43afb2767e9528c0ddab20065cce531108588e11da48c1c679625","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · scicode"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-low","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Low Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"02c94d91fc1a8045e9e4f6eedbe69c2e3ec19e31b8e2d2dd8a26d9d5226ff5e4","result":{"confidence_high":54.936309,"confidence_low":43.464651,"sample_count":288,"score":49.189815,"score_display":"49.2","source_stated_rank":null},"run_fingerprint":"8fb799ae73943e778ceacbf1a7b69e6097d45a40863edb5ecb9d38c6015c3880","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2ec7f4efc62b7f5cbdefb881e93af1693be002f77508c2eaa5f6d45cd569cbcd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"1.890512","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-5-high","source_effort":null,"source_model_version":"claude-opus-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"f5d0d49ca2d1395a9572a75707f8f175364d159218fff2ad97b2fb9f4823395f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2164.57,"score_display":"2164.57","source_stated_rank":4},"run_fingerprint":"ca15296d64c434fe4c6c93dbd9b32c40a7cdb82f6706a770d5b21096332f9168","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-07-23","status":"fresh"},"measurement_id":"c313f02b3c4887ee9c65772d599e985d20780c9fedd69918ed669935a42f90d4","metric_details":{"benchmark_page_updated":"2026-10-02","cost_per_test_usd":33.876235,"date_is_proxy":false,"latency_seconds":5301.22,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-opus-5","source_locator":"Results · Overall accuracy","stderr":3.001},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":null,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"bae5570e4d9b683e073ff86643ab4b9f8f9ce3387a2bdcccecd9c1100b98da97","temperature":null,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"2a33330c0703bf2df3470905287b646775311977e13466aeccb979a8cad73a0b","result":{"confidence_high":94.28595999999999,"confidence_low":82.52204,"sample_count":null,"score":88.404,"score_display":"88.4","source_stated_rank":8},"run_fingerprint":"b5df8e319198ec136153417f0dd6118ea5584bad8170d53436ba4398e48ae714","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ed92784a3a69757d5a7d96f7","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a37432b889b0d27ab7d4ce71f446957d8ab997cb640852c3aafdf8e9c4caf2d2","metric_details":{"license":"Proprietary","variance":10.906880273449918},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-5-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8ae2191850916ecf98f33d925b0a649da921113ec523949123d61642202eb41d","result":{"confidence_high":1701.4528667005131,"confidence_low":1688.5070826095127,"sample_count":16954,"score":1694.979974655013,"score_display":"1695","source_stated_rank":7},"run_fingerprint":"c664de2ce9cee5f0e2b6280fcdc6abacad0871fb943d6af7e054935e669e8f20","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ed5e48288df62aee6ec4698297065b71c9de0b8d229a0ce1de011d1fb0ccf2d6","metric_details":{"license":"Proprietary","variance":8.204902501258077},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-5-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"dc4ccfcf8eb1e043878ae789b1c53e0baf8068bded8d4eb70b47c0530061199d","result":{"confidence_high":1665.5841056456745,"confidence_low":1654.355785185981,"sample_count":21506,"score":1659.9699454158276,"score_display":"1660","source_stated_rank":12},"run_fingerprint":"783f8da19be2a1f0619d778b9366252f28f3baf9dd5b0962216679d76b270859","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aa525bb0b104cdf0c4b4248db5b0afa55e3dddec38280cf609fe3063556351b5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"2f995c5ebbbc46c35cc022166f865c7a4c8bf44050d54d53997415f965051b0b","result":{"confidence_high":1694.48,"confidence_low":1679.51,"sample_count":12087,"score":1686.99,"score_display":"1686.99","source_stated_rank":8},"run_fingerprint":"487e298fccf7f256667ce8f4c80534cba25eaec67be5570838b345bbb164c9cb","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9585d44f55330b6a0e437dd99cc369623da6d363aed3ec668323adb29ede9bd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-5_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"6978c65394ce3c0bd63a6e6b7b96e470d9bb97542d830bea10d9d86156c936ea","result":{"confidence_high":1667.4,"confidence_low":1653.5,"sample_count":12566,"score":1660.45,"score_display":"1660.45","source_stated_rank":11},"run_fingerprint":"4c780b9d737d75559a07b89f75be8a19289711572dd3b6e146cde8b5860f6dc2","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1097074c28d3b3978c095db25d9f55d4880b08276304bccb7854d2ca2e05cb0a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"329bc9c59b122dd43f418ca50148d734057d92ea5b39e2b294aa783aeba0d7f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.78,"score_display":"91.8","source_stated_rank":7},"run_fingerprint":"4ee2bd6af968fa9a54ad39cfd2dfabe384685b50ad4e0d5cd5db9c63c0850c0a","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"df5c34cfbaf3b6f1e07bc96409303edaf1b7437e1feadd631687aeac5f011678","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"c34a2fd41068b4693a6a352760b384ab1dd06d187b05a20f39fa2cddfc38708c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.59,"score_display":"91.6","source_stated_rank":8},"run_fingerprint":"2e359b02ba0e7f5866a388044aef786f04f72751ff2a832a392d4fd063677b8e","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d6129b58d223982ca26fdd5c83583ab51f5a95aeaff6cc34903fb9bcfb26e3ac","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"6e86b0a53b2eee7961b65c8e18a8204d5ba3a9b459481153b85892802d727378","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.34,"score_display":"86.3","source_stated_rank":13},"run_fingerprint":"88a917ecb69ed41e332a329eff91df0178460b0c2b6b2e3204a78d234edd2346","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dfccb277c22138561e21c2ab","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-07-23","status":"fresh"},"measurement_id":"5ab643219d851390983a725f631e9692114b0274ecddedc48a9afd5dcbc7dabd","metric_details":{"benchmark_page_updated":"2026-09-28","cost_per_test_usd":0.886936,"date_is_proxy":false,"latency_seconds":752.932,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-opus-5","source_locator":"Results · Overall accuracy","stderr":0.991},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"bae5570e4d9b683e073ff86643ab4b9f8f9ce3387a2bdcccecd9c1100b98da97","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-5"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"0151f009624f5ee0f0c6fc09dfc0073244ea22e2319c91bf3b377f0d09e222b8","result":{"confidence_high":86.58636,"confidence_low":82.70164000000001,"sample_count":null,"score":84.644,"score_display":"84.6","source_stated_rank":5},"run_fingerprint":"ec4e1ddc70fb0034ee7feddf7e475ac8130a3fbaf08cbb66e9793b7bdf77998b","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c2e5411e22528a4a0041616d62b4361f997d8ab32f19c323d9a30fc96776a005","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · terminalbenchV21"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-opus-5","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Max Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"6b2df6c907e274015d3d229866b220998f7c3ba6d9e1aae6f86834a886252f91","result":{"confidence_high":94.052379,"confidence_low":80.985823,"sample_count":89,"score":89.138577,"score_display":"89.1","source_stated_rank":null},"run_fingerprint":"9c58f2d3ac4bece3fa43f48903ed0ec169ecdfbd1546bd49f85484f206808701","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d76b45325b5c0e5967cc842eedd5eae16ee3f95e2a3585770f0dc5b64a7c0370","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · terminalbenchV21"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-opus-5-xhigh","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"e90e8bf175685856b0691ba429e6a036155671f78637245e8397c053c0a63777","result":{"confidence_high":93.233339,"confidence_low":79.650655,"sample_count":89,"score":88.014981,"score_display":"88.0","source_stated_rank":null},"run_fingerprint":"d34d525b253b2b6a9fefad8bf8cb1a2314f5c4c943e5e55d284538dab1e68f12","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5fec0849fe095a9ae00b330bf9dbc865d70c7288f22d8e47b4562b996cf3ecc8","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · terminalbenchV21"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-opus-5-high","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, High Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"ee464faf8244131ecd20580389de701435a976d5a8991e1a73763682f48e5ab6","result":{"confidence_high":92.956555,"confidence_low":79.20937,"sample_count":89,"score":87.640449,"score_display":"87.6","source_stated_rank":null},"run_fingerprint":"95dfddcab1003c81f4ec22a965d67d95966e29fc80cb637b147b115055d54089","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"11fed132b10618b06818524094ad0855d500fd887d32a4fb8d21d676dcf68afb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · terminalbenchV21"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-opus-5-medium","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"627468ff1d6684e74c01b49bacb19de4e6d94b952a22973238d1f0b84953a906","result":{"confidence_high":91.832281,"confidence_low":77.461369,"sample_count":89,"score":86.142322,"score_display":"86.1","source_stated_rank":null},"run_fingerprint":"5087daa532b42f6ccfe848bb444a5a7f6eee23b9d41f58be8f14e9212a75ae34","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"50f858b61dbf958aeb966c7d546702adf2cbb65d0c8628726c962e13c64ed055","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · terminalbenchV21"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-opus-5-low","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Low Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"1afad98a64aa2dbdfaa3eaee70c275cf97a5ee52e43f672faa03bfe71ea74241","result":{"confidence_high":84.017678,"confidence_low":66.606179,"sample_count":89,"score":76.404494,"score_display":"76.4","source_stated_rank":null},"run_fingerprint":"47c1463be16be124becb51733f8bf1018d6557d21e694cf9afeb4e917e455470","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T22:08:44.699364+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T22:08:44.699364+00:00","status":"fresh"},"measurement_id":"c3d9b14837df09d99fe6907ac42c1da03a61d742061c11370e38024406950436","metric_details":{"accuracy_stderr":1.58,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:07:44.976503+00:00","total_cost_usd":5818.2,"total_tokens":7280393172},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"Princeton","url":"https://www.princeton.edu"},"agent_url":"https://github.com/SWE-agent/mini-swe-agent","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"b26434fb-f08b-4f95-bcc5-ce0c7310e047","model_release_date":"2026-07-24","model_url":"https://www.anthropic.com/news/claude-opus-5","n_trials":370,"source_row_date":"2026-07-24"},"run_count":5,"scaffold":"mini-SWE-agent","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"5131b7fcd81347976de3f3afe9b5af9ea833e46f36bafd8cdc477f8e699e3974","result":{"confidence_high":54.054747,"confidence_low":32.065785,"sample_count":74,"score":42.7,"score_display":"42.70","source_stated_rank":1},"run_fingerprint":"6871dca2fd8dd990bd67c2b29d8582c79618b751b4aaf7858a370ab31ad98dca","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":63,"at":"2026-08-03","basis":"evaluation_at","evaluated_at":"2026-08-03","first_observed_at":"2026-08-28T07:09:05Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6b6303802e6b5665537e0fdb3e687db4ec4313d75fd7eb7f1f13004317e26f57","metric_details":{"cost":null,"pass_2":39.175257731958766,"pass_3":34.5360824742268,"pass_4":31.958762886597935},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"claude-opus-5_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"b506c9b684e54c4990e42c89c90ca0191030f4ede4e7efce735f197e2f9ed3e8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.71134020618557,"score_display":"48.7","source_stated_rank":null},"run_fingerprint":"b2e56844a20f5b974664ad968064414c521b99cf9864be4b78dae3b6858f586e","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1ad5dc2d286820a0e524ef9dedcbcbd77824a87ccf082503a7566830369a266d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · tauBanking"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Max Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"154f156c90ea702c55f0088b80f24f0bc7774ad5213995666b58947943052ec6","result":{"confidence_high":52.004251,"confidence_low":32.724274,"sample_count":97,"score":42.061856,"score_display":"42.1","source_stated_rank":null},"run_fingerprint":"29519ff8c1a2f1c4b11ed1a5ba1d6a524f05d7ecd4f3484ccaa730503d9b9391","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"81124648b11d21672310755740286c1975bdacca61219a55ef3c57295b54f6e4","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · tauBanking"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-xhigh","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"be3edfed55c323cf98afe2bfb4f56096c94f9c650d45334b6c99bafaa9358ae4","result":{"confidence_high":53.228596,"confidence_low":33.879899,"sample_count":97,"score":43.298969,"score_display":"43.3","source_stated_rank":null},"run_fingerprint":"f7940a8b3d7f0f54b2ea4a08ed44352691d92a7394170be18fb6014616b65a12","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dbbdf9cda5ec92ede9e332dbf4bd06dc79fd79f4d7ea6c14a4e7842968097797","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · tauBanking"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-high","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, High Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"44c48a5c95d55bbd165fd4bcfd2b7ebda60586d682694e5abb5fd2150289c797","result":{"confidence_high":54.649545,"confidence_low":35.235582,"sample_count":97,"score":44.742268,"score_display":"44.7","source_stated_rank":null},"run_fingerprint":"2ae1fb0cfed95748aa956ab7beee318804b37e66e1edf2c6a4db4b29eb91535c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"94df808afcce8d2851f4488b99b42d37e23ec8df311a649ff48adef992c4c72a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · tauBanking"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-medium","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"39334372e2cd9198356c6d298d595cd5f39af983eff4f4ac8a5ec302e6f504a4","result":{"confidence_high":48.50263,"confidence_low":29.482645,"sample_count":97,"score":38.556701,"score_display":"38.6","source_stated_rank":null},"run_fingerprint":"ceca46690d8ec84468d56b3dabdb15fa640050c66bc4095ef39839eab10150c3","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"83c91cd11c51ac6b134b556607042503ba0f931a6ae1cb4e9f34063dbcf9f293","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · tauBanking"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-opus-5-low","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Low Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"f531859ec11d8e31c0a68f6bcee55e7ff6ba39de451e1e414403dc7a91c29c15","result":{"confidence_high":40.061106,"confidence_low":22.057701,"sample_count":97,"score":30.309278,"score_display":"30.3","source_stated_rank":null},"run_fingerprint":"b68a3829d230f2d54bc0a2dbb77d33f494a959aea6e50ba886a6cec991a4632a","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":62,"at":"2026-08-03T15:52:23Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-03T15:52:23Z","status":"fresh"},"measurement_id":"3ceaffb8f04343848e6330dc38e753f1aeba9918395be5f8d879e472fd46c342","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.1,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=claude-opus-5 (xhigh); createdAt=2026-08-03T15:52:23Z","source_row_created_at":"2026-08-03T15:52:23Z","task_pass_coverage_threshold_percent":75},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"claude-opus-5 (xhigh)","source_reasoning_label":"xhigh","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"b24a53faade197a830d5ff4bcd6e6d027fa0e248dc5ea1972ca1838a2e9ed380","result":{"confidence_high":87.89999999999999,"confidence_low":83.7,"sample_count":1000,"score":85.8,"score_display":"85.8","source_stated_rank":2},"run_fingerprint":"0ccf529323751a203fa2850600eb09ad51613991136ede138abf9a1ef232912a","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=claude-opus-5 (xhigh); createdAt=2026-08-03T15:52:23Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6470b94c82a9f62ac9ba96628be3f94853d9da338901c94489e7b092511911ee","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.77","mean_standard_error":"4.0","model_release_date":"2026-07-24","notes":null,"standard_error_points":509.99999999999994,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 5","source_effort":null,"source_model_version":"claude-opus-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"bb75b82c0964c27de4324f89df4d994019582287fcc339a702417303c909fa16","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":65.8,"score_display":"65.8","source_stated_rank":5},"run_fingerprint":"5759d2a72935194429c479b3bb6838f2b77e20d3fcf877f960f48280e63ec87d","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"35629ea866bab3d02cb8a5a67c4f74f09d9b765470db47ea8d63f26d43b0c951","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":22781},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Claude Opus 5 (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c315d4f6b41cb189e1340bf675715f78023aa90155398b4e4ec67134c6d5e0f8","result":{"confidence_high":0.09398347689954321,"confidence_low":0.06448680051842494,"sample_count":3006257,"score":0.07923513870898408,"score_display":"0.0792","source_stated_rank":9},"run_fingerprint":"cbd740f68083330e4a70e66af8bb178998b09e0df38dcc657832601af978fae7","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7d27b24e2f73d5cdac57bcba0bf122e1ab20513e6fe097d4152aea2b27165bcf","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":27560},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Claude Opus 5 (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7d931e7785e1bbe15f6fe3f1bc351d11bfdbb6308c0dcd821642c533426c0354","result":{"confidence_high":0.10080670137578338,"confidence_low":0.07261671714738879,"sample_count":3531760,"score":0.08671170926158608,"score_display":"0.0867","source_stated_rank":7},"run_fingerprint":"abf929406c0e03c957315a87ff92609397e8fe1fb4e51486199e19862cbc68f1","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b7406152b66a229c04d6f89ee9d2e8e808c74caf070aa28671b0003736b8e674","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5","source_effort":null,"source_model_version":"claude-opus-5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"ef5249d0c3264aa5f421a764813869371b4d544ebde5029afcbb4ab79490bf64","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":11181.87333333333,"score_display":"11181.87","source_stated_rank":3},"run_fingerprint":"0975440adea67cb7b0d2b398317d650abb9e478d5b9ede1006957987209878a5","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1e112a485e54e7367e8e34df","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"49948b02705dc8058dfab7a1ed0c2376fdb8b1b544534af5278f51c40f61bdc9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · gdpval"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Max Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"b24f93d9b1d18633446306a605f0432baaaa7ddf10f440eefe02d2a7b3b3ee8e","result":{"confidence_high":1732.95,"confidence_low":1682.9,"sample_count":220,"score":1707.92,"score_display":"1708","source_stated_rank":null},"run_fingerprint":"ac90b912b14f5f883e8111283563805adefe209aa1ed52be50ff965e4eb89ff2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Max Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"043bfb851b2c1f64217ef5b59f766d598bd716b1d6b902cf74b6b902a8e7ee38","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · gdpval"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5-xhigh","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"81d3e8bf51fe257303b53e2ced088b58eab9ecec854abaa4957fce611592d45f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1676.3,"score_display":"1676","source_stated_rank":null},"run_fingerprint":"5531f117a3edf93befb2a0126ea13920552cb75bc4ecbca65e21ce8c6dee39b0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Xhigh Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"47c56ea08ed8a36108c8d502de69df33ed4e7f79ea4ef4331ee03f60e0d85bd4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · gdpval"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5-high","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, High Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"4c48ce84820543fc6763b79a79337df9448a39fbdbcabb9d3bf95338df233ea9","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1581.0,"score_display":"1581","source_stated_rank":null},"run_fingerprint":"af43296b9b8e01d97544975376a748ed08dd62efe3fd1dad0a5bd3dd67e87c3b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, High Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"98eb78b6042b49b772775b06b6b283ec30aad5b20cd0aac53799567b6d171321","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · gdpval"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5-medium","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"278c12ab2af84a9b68a68e8d564174ac8fa03f6ed9f6bfb5a3986dddd5ac8216","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1475.93,"score_display":"1476","source_stated_rank":null},"run_fingerprint":"26fe2041246be43e14689ab29552d6c5ecda50802c0fca2cddd1fa9e98d74c01","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Medium Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"34f95488dcb48a4a37b65a51482efae56ec91a03085cd5ab9cf167fc5e73d222","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-24","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · gdpval"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-5-low","upstream_model_label":"Claude Opus 5 (Adaptive Reasoning, Low Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"7797d795b759a787823dc065218821a13fff8bce6bc8b18681eb1aa29d8f6eed","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1293.66,"score_display":"1294","source_stated_rank":null},"run_fingerprint":"773f3d0eeef3babdf68c84d5f3cf0e7999ccda41b8b1c124e07cf3b79fd873c8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 5 (Adaptive Reasoning, Low Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:40Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:40Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d8b79b2185b7e4738b60e63187f51c0d13a4a88b62bf0461bb5705b836d874e6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (max)","source_effort":"max","source_model_version":"claude-opus-5_max","source_row_date":null,"source_scaffold":"batch tool","upstream_leaderboard_url":null},"run_count":null,"scaffold":"batch tool","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"7b8b7a1ab940a3e15ada25e67e664fc8cc3cd93e799a1174dfb120fba783b039","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.430000000000003,"score_display":"31.4","source_stated_rank":1},"run_fingerprint":"ea97680975e285c878d32c9573833121641b6ebbf221a825b7d1fdadf1d9570e","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_892e19ae915f3f7411f86632","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:40Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:40Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"72f350ac1fa28dcbed3c6b3842984890a3b5edf741a43af647bdbc2d7f75d0fa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (xhigh)","source_effort":"xhigh","source_model_version":"claude-opus-5_xhigh","source_row_date":null,"source_scaffold":"batch tool","upstream_leaderboard_url":null},"run_count":null,"scaffold":"batch tool","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"caab9ed04a041628cf8dcf524f758057cc9895e076d4c01e6a9ca1dd8e642d6c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.210000000000004,"score_display":"30.2","source_stated_rank":2},"run_fingerprint":"32fad4e55a30610891912489ad917c59c38f546da86dd75acc615cdeb2a8723d","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1f510fcb6cca3743ad871cce","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:40Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:40Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"209e2e7bd480bd4b012e75df1728a29acd1f79cff5358c6c4fe7162198daaed4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (high)","source_effort":"high","source_model_version":"claude-opus-5_high","source_row_date":null,"source_scaffold":"batch tool","upstream_leaderboard_url":null},"run_count":null,"scaffold":"batch tool","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"616f0bfd037cdfc4755102ec9a3c4653ac5e8100b3c073e7dcb03c7d41782f0e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":28.98,"score_display":"29.0","source_stated_rank":3},"run_fingerprint":"fa20aea5e37d8ace8927b151003f8ccba136dde17a0ffabc40e44575b63c075c","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_39a30289a15f1db029022e6c","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:40Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:40Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"15d5e158485051e1408537242fca06c9f490400a208d882ba36138d0198d73ba","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (medium)","source_effort":"medium","source_model_version":"claude-opus-5_medium","source_row_date":null,"source_scaffold":"batch tool","upstream_leaderboard_url":null},"run_count":null,"scaffold":"batch tool","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"4ff2a1b1b51563d19978001dd67d2449ea48b2e38f588fa319beb69aa333cd71","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.33,"score_display":"25.3","source_stated_rank":5},"run_fingerprint":"fff3ad0a9721a19916c27fd2bf07b7726fac0dffbcbfd4be078cf78f2199a81f","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_7755c5bbeb7e47ccfd5ac6c4","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:40Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:40Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"84aeb1e736240ff274b81f39a5c2264929b1fcd5f3e0eb4ab80df89503bdb079","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-24","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-opus-5","name":"Claude Opus 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 5 (low)","source_effort":"low","source_model_version":"claude-opus-5_low","source_row_date":null,"source_scaffold":"batch tool","upstream_leaderboard_url":null},"run_count":null,"scaffold":"batch tool","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"54e130bffed28925cce66157b79f86174b16c95572055db181efc9abc05df1f4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":22.32,"score_display":"22.3","source_stated_rank":6},"run_fingerprint":"e290f6c8cc39ee25e5c34192c81d1263f00666d495987c57a34d2ea73bbf3c20","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a6beeea9190e94a26ed18cbb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ce73c6d3d307c5f59f3a1ba4049cbb2c8cbc4bcf6fded7400048a183cbe71231","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-23"},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.5 Pro; model release: 2026-04-23","source_accessibility":"API access","source_model_name":"GPT-5.5 Pro","source_model_release_date":"2026-04-23","source_model_versions":["gpt-5.5-pro-pre-release_xhigh","gpt-5.5-pro_high","gpt-5.5-pro_unknown","gpt-5.5-pro_xhigh"],"source_reasoning_efforts":["high","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"63d60a170566e6e95568fa76e25b044f1ca8c2a459dec04c888f082feb78bf40","result":{"confidence_high":165.99,"confidence_low":159.03,"sample_count":null,"score":162.38,"score_display":"162.38","source_stated_rank":6},"run_fingerprint":"83ecc3f2c7bfea314afd91258fe278e51e9eee70122068b48822a5b0dcff73ad","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a898b81cd13e7c0a7333367f","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":163,"at":"2026-04-24T16:01:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-24T16:01:38.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"d9a2600d99c015de615ab15ea56fa416acb677738fc27f6c92ef83ae3206d93e","metric_details":{"best_score_across_scorers":"0.9392135642135643","model_release_date":"2026-04-23","stderr":1.5832823810775416},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"cBmhJzrC3SkfZ7uVgGVYZX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcBmhJzrC3SkfZ7uVgGVYZX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/cBmhJzrC3SkfZ7uVgGVYZX.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"8a796135e8622378052adc7a4bba7c6aca7c1487fcf5796012bf357caa9a8c02","result":{"confidence_high":97.02458988826841,"confidence_low":90.81812295444443,"sample_count":null,"score":93.92135642135642,"score_display":"93.9","source_stated_rank":13},"run_fingerprint":"b13fd5472bab54da375af4bebcf42e8ae984fbbf0015f01b88731166c2bcf8b2","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":163,"at":"2026-04-24T23:38:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-24T23:38:07.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5dad8b67be8c9f4464c568b5844b5827193ac2369f2f682397fed7439f3d317c","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-04-23","stderr":0.0},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"YxbseUDNgFUjokfYXui62r","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYxbseUDNgFUjokfYXui62r.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/YxbseUDNgFUjokfYXui62r.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"30d980fc451103bcdee4e637b703d5d5481608163460c228f5ae952e8a48d05c","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"94f704b4b978301ca72b139d074bd60a7718bda5c61ed33b6071156206087263","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-12T21:33:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-12T21:33:12.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a4ced067194b19a7f0b316d6c1b603921bb4dba6cf99c1b722abca9347fd06fb","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.8771929824561403","leaderboard_private_problem_count":285,"model_release_date":"2026-04-23","public_example_count":10,"stderr":1.9476010341530077},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"LSKDetgzyVojru45w7RJhf","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.5 Pro (xhigh)","thinking":null,"upstream_model_id":"gpt-5.5-pro_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"4f7bfe326c1ca2ae446294e4eb0445dfa7589ed6b6ed974b7c4a84b746e164e2","result":{"confidence_high":91.53659627255392,"confidence_low":83.90200021867413,"sample_count":285,"score":87.71929824561403,"score_display":"87.7","source_stated_rank":8},"run_fingerprint":"16312dd948b493e31f87587c1b81bb5ca7c0fae244f2a8247176cac70a7a911f","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3b728b7c53feb530171c2eaa5af662091432edfabd12138a2d17c4a8ef491a52","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":10.76,"model_release_date":"2026-04-23T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-5-pro-2026-04-23-thinking","model_type":"CoT","upstream_model_id":"gpt-5-5-pro-2026-04-23-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5fa4ab1dc988b73dbbbd3e7038300348d2ebb3420f14d96e9a745555386e7758","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.16,"score_display":"84","source_stated_rank":33},"run_fingerprint":"ea5cb8842e1c18463946a66ce2794e80d532a8154b10e38a9b26ea9a98d1aa55","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"512c7c0e3dfdb979ff3c2a48581f733d1a27801153406f3e8b5c985841da0cdc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":10.51,"model_release_date":"2026-04-23T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-5-pro-2026-04-23-thinking","model_type":"CoT","upstream_model_id":"gpt-5-5-pro-2026-04-23-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"562294f168735d2d3b9559c887733a8b653f9aaa2fea24a784a2fbf303466d8c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.58,"score_display":"85","source_stated_rank":31},"run_fingerprint":"e7e7697d9a2f59a2ffb3a6f48d0717486089dfac2bf43669b24c44e46d61f69c","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_2ed771acdbb41d4dbb230330","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2d07ab7d23eb899e1995e52d5397968cae2d347cd035e421a48bc631b132777e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"10.76","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Pro (xHigh)","source_effort":null,"source_model_version":"gpt-5.5-pro_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"7b225f8fb84009efe01bc539c1c45475d6bb201c476da1a0f819c5372a4e999b","result":{"confidence_high":87.56834,"confidence_low":80.030307,"sample_count":360,"score":84.16,"score_display":"84.2","source_stated_rank":34},"run_fingerprint":"162fbcfaa89c4afd51a897a5b1f51428c23525c008488612dbf8899ae22ee9b1","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c45955163f34a270a06cc54d090127f457ca9b4126c186c29992e2a52ff0cd4c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Pro (XHigh)","source_effort":null,"source_model_version":"gpt-5.5-pro_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"63c43406c0c28feca7800197151070f52b4dea271a10318b1283cc07d26ae697","result":{"confidence_high":87.56834,"confidence_low":80.030307,"sample_count":360,"score":84.16,"score_display":"84.2","source_stated_rank":34},"run_fingerprint":"312720ecc018f8260e3e9c06ab2d4b53642fd38675a70e347f8ad04f19491b75","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"45f56c986401e553cdd68b83560905e07f0eb7cf803a3eb99ac5a392b5cb3207","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"10.51","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Pro (High)","source_effort":null,"source_model_version":"gpt-5.5-pro_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"636388b2811c71a27f824de92256b29ba4b872620e4b1afd335dd3e852d9bd50","result":{"confidence_high":87.943679,"confidence_low":80.486099,"sample_count":360,"score":84.58,"score_display":"84.6","source_stated_rank":32},"run_fingerprint":"e5d6374c483bea13add11d677c1eb54a30b8e7ea40c69aac8970934bc4394415","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_2ed771acdbb41d4dbb230330","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-12T21:33:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-12T21:33:12.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"2b4cfb7a9722adcbf8463b0717afd74ebab7155c4138c10d1cb109f762b628fe","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.7804878048780488","leaderboard_private_problem_count":41,"model_release_date":"2026-04-23","public_example_count":2,"stderr":6.5445892024384085},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"SFKXCPUu73mwxMoUotiZM2","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.5 Pro (xhigh)","thinking":null,"upstream_model_id":"gpt-5.5-pro_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"3413c81f5619400532d2201f988104f1eb7009ba28c73c1888d3e790ea055219","result":{"confidence_high":90.87617532458415,"confidence_low":65.2213856510256,"sample_count":41,"score":78.04878048780488,"score_display":"78.0","source_stated_rank":13},"run_fingerprint":"68f26a519337d52c437da89a8d4af4a3b66aadc751094d26a325151fec37cae5","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"664e786a126a4cc93a75f446e1afe3424b15c2d039588d133da0a864061444ef","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Pro (xhigh)","source_effort":null,"source_model_version":"gpt-5.5-pro_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b0f85254d1f766d02c983eed82e57ffce83084361eb30366b0de23cc568247d7","result":{"confidence_high":42.054086,"confidence_low":21.083298,"sample_count":71,"score":30.571428571428598,"score_display":"30.6","source_stated_rank":13},"run_fingerprint":"7d4e956ab1339c71f2968e0793ba8612b786b4143619a2e05e38af9ca197b54a","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d4aec8f55422a80813f4356189717f239238b4506e51def8d77a6c9a9bd2db25","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":922000,"is_open_weights":false,"is_reasoning":true,"model_release_date":null,"source_locator":"Leaderboard models table · GPT-5.5 Pro (Xhigh) · critpt"},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-5-pro","upstream_model_label":"GPT-5.5 Pro (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"3894f3a27e03620727785e0ba970ffba70923ac101d5e1f720f18c9bd544230e","result":{"confidence_high":42.138999,"confidence_low":21.025397,"sample_count":70,"score":30.571429,"score_display":"30.6","source_stated_rank":null},"run_fingerprint":"64ede397263110d65a612c5ed072df4bcb093886af0f59f8ba1c9298f5cca641","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.5 Pro (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b920231c832df80a9d032bc9f3531a68ea6b1158a6b455a1206f9076ac6e53e2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5-pro_high","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"8253bf31f96ae1c9afe2d99ed7682acbb2c7f46187d7f9ccae405ec8d28f9861","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.9,"score_display":"76.9","source_stated_rank":4},"run_fingerprint":"f25bf1a0438a943f41d5c42ac6f278a10a94c054234c8d955ddb18ee2b92ad32","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_2ed771acdbb41d4dbb230330","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d1aa3bf787d10e1a946d4d85ef321a9e9ee8a466e54abe88c2c67ba6ad79226f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5-pro_unknown","source_row_date":null,"upstream_leaderboard_url":"https://lmcouncil.ai/benchmarks"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"cfdb7d5ec47c99ae67cd33c653a000aee834cf8e24e1a16e148f3bcaf31e5952","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.9,"score_display":"76.9","source_stated_rank":4},"run_fingerprint":"d611b4e9358363c75aa7d3d34eaf54f988d8e82846f7393e5871cb482a95e65f","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4b78dd05f228eebefc5e35b1","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":157,"at":"2026-05-01T12:33:37.643Z","basis":"evaluation_started_at","evaluated_at":"2026-05-01T12:33:37.643Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"024ccfbd1262a4baec9467b474eeb7cbfe9302ade2938509e63267f756f6800d","metric_details":{"best_score_across_scorers":"0.64","model_release_date":"2026-04-23","stderr":4.8242},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gpt-5-5-pro-pre-release-xhigh-chess-puzzles-frankenstein","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ec10d38737b86fe16701529392d2770027688ad5ee82196099f6a7b41a1ecb08","result":{"confidence_high":73.455432,"confidence_low":54.544568,"sample_count":100,"score":64.0,"score_display":"64.0","source_stated_rank":2},"run_fingerprint":"392be4dd221719e113a3f8cc98c0d6b9dd75cd46d799400edd957e770e3a2667","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"16454667d728462dd7367a1a0ed8da6dac5957f9c48d6f29153e2f4753b22f32","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.52","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Pro (xHigh)","source_effort":null,"source_model_version":"gpt-5.5-pro_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5c55a0d14a562b4baebbd7babd6ce742dc336a3fbf3083fd945dff3182317256","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.0,"score_display":"95.0","source_stated_rank":33},"run_fingerprint":"7d6fc937ddbcc3946105fb7ce3e97f25f9e385585ff8d3782015c53dda08e6b3","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a696e79e2068a807cf28f16acfaca0a3cf078b26a7c8bedb233a51b3558c3e8c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Pro (XHigh)","source_effort":null,"source_model_version":"gpt-5.5-pro_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0c7ddb694d67dd61de1222bbfba0f23de517c2907a5f12313c6734e101461353","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.0,"score_display":"95.0","source_stated_rank":33},"run_fingerprint":"c2229b2453a58ef80cb13f0d8083eacce09b9b278325d2f4aa0efa69a29c356e","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_52423dacb65ae6d97e85c24b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9573e21f17fd381533b79d2dda1022ddc7d43f7789fac939e97eeecce2ce71e2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.53","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-5-pro","name":"GPT 5.5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Pro (High)","source_effort":null,"source_model_version":"gpt-5.5-pro_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"4d374b255b2a27ef18cdb00e11fa5361a64b06f61e305e18a33a5f580ae61937","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":96.5,"score_display":"96.5","source_stated_rank":21},"run_fingerprint":"2796e858bfffaffa6fdeeb2508a874ad69c8a85edaad56494135f339f6477772","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_2ed771acdbb41d4dbb230330","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"03d8bd8a2ba06703bb48821472bd246d42d2430fc9bd65a34a899137ac580b74","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-06-09"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Fable 5; model release: 2026-06-09","source_accessibility":"API access","source_model_name":"Claude Fable 5","source_model_release_date":"2026-06-09","source_model_versions":["claude-fable-5","claude-fable-5_high","claude-fable-5_low","claude-fable-5_max","claude-fable-5_unknown","claude-fable-5_xhigh"],"source_reasoning_efforts":["low","high","xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0299309d1f5153df3371bceb97380ba7d966210ed5dd29c1f758079d8c4f9c76","result":{"confidence_high":166.11,"confidence_low":159.61,"sample_count":null,"score":162.22,"score_display":"162.22","source_stated_rank":7},"run_fingerprint":"a18194146bba6a5321be954f54b475c32904bba788b9f73378b181b9ce56a2b3","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d040556a661041609251c38e","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dc44984d912f28abdd19c58d61b5fffc388149a57f0d25fa160c0d27a9bebb99","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · intelligenceIndex"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5","upstream_model_label":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"da4f267300a06550be17dc35403f0c50e430929f94d254b788eb4b18e0d9c6d0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.699445,"score_display":"49.7","source_stated_rank":null},"run_fingerprint":"260809cafbf8f093dc10cf9e38a51b92c398f3c87ab7ce3a15003c5168bb2977","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_a31432cd65f025d42eaa2c1e","provider_config":{"source_label":"adaptive"},"provider_mode_key":"Adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e124b340761a242ee6893671b3ebe5bd60197c2adb399bf33c7c07aef8b85779","metric_details":{"license":"Proprietary","variance":5.015174048102348},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1508.6602479313276,"confidence_low":1499.8817322061268,"sample_count":38387,"score":1504.270990068727,"score_display":"1504","source_stated_rank":3},"run_fingerprint":"74bc84bb648f50c55ae22e23605ea399814b895b44a8aef76bda9ca6cda28cde","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3b31ce898b9f332fe6aecbf6d4fb6d75feeaeac69128ed6d8ebf5c026aecb018","metric_details":{"category_scores":{"Agentic Coding":62.17166666666666,"Coding":85.992,"Data Analysis":80.53766666666667,"IF":75.77074999999999,"Language":90.68400000000001,"Mathematics":95.98575,"Reasoning":89.65375}},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.97079761904763,"score_display":"82.97","source_stated_rank":3},"run_fingerprint":"b0378cbfbd79b3c84438f5ba2f447ac56730f31fa03e025f5d5e6a75c645c32f","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"86e5167113f4f23bdd41beaeae65fa831cd9e924261b5330926262a3e62a41c2","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · hle"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5","upstream_model_label":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2b66769f3d0af076fa2b929944b6ca4b078d46e1e57f4168d926863422698241","result":{"confidence_high":57.553416,"confidence_low":53.363203,"sample_count":2158,"score":55.468026,"score_display":"55.5","source_stated_rank":null},"run_fingerprint":"e41d0e4fa6b05f1fdc571b507b34bcfc035a7e4397bf1413ced61acc76c36886","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_a31432cd65f025d42eaa2c1e","provider_config":{"source_label":"adaptive"},"provider_mode_key":"Adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T23:44:03.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T23:44:03.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7bbf5f81e1ba4a9970c9be12f3fb45186f6d4a0c11a8e88b278b5b4c62b76121","metric_details":{"best_score_across_scorers":"0.707","model_release_date":"2026-06-09","stderr":1.4399942998441293},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"SEnvgtunRa5GwAtU2sHa87","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"049ccdbfb7201fed22bb9df66ab7dc0362db3395fe5f74a70bc801bec87c0e59","result":{"confidence_high":73.5223888276945,"confidence_low":67.8776111723055,"sample_count":1000,"score":70.7,"score_display":"70.7","source_stated_rank":6},"run_fingerprint":"1eac34c34ae7ec76020c22347c38141504b3d89f13215207d88149277a405f5f","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d52a4bf6c494a0166f10bbfd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"815284a4d8a3f5af08bc61da0c9716cc545c8684258d0bfb6920712fb4462816","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.450473,"date_is_proxy":true,"latency_seconds":99.902,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.939},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","few_shot_accuracy":92.929,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5","zero_shot_accuracy":93.434},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"ea78ecd2f2860042714b4910c1d0fd0bd06500e9842651c3623089c0c8bda90d","result":{"confidence_high":96.98244,"confidence_low":89.38156000000001,"sample_count":396,"score":93.182,"score_display":"93.18","source_stated_rank":10},"run_fingerprint":"089adc5104e574c59f4b6b9dac2e3f5a977bc763330bf7ac42822e060fdc78e2","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cf1b7d9cb481284e90f3147611e3c44051c84b157e19ea09839640cad2fa7b74","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · gpqa"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5","upstream_model_label":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5606c8df4c3741304831fa6f918c57022e1009d85dd86af6285015286e293c19","result":{"confidence_high":95.510587,"confidence_low":88.119349,"sample_count":198,"score":92.626263,"score_display":"92.6","source_stated_rank":null},"run_fingerprint":"1bdbfb91b2ca1e0435f4e496f943187aa90b16184acfd491acb417e297fe0cdb","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_a31432cd65f025d42eaa2c1e","provider_config":{"source_label":"adaptive"},"provider_mode_key":"Adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:17:56.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:17:56.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"748ff35a7f07bb8678b4c2e3df5181016239ccbe0d998d9a9199f6ee1d2cac2a","metric_details":{"best_score_across_scorers":"0.8585858585858586","model_release_date":"2026-06-09","stderr":2.4825909793343355},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"S5pm7ctk9zgPfbyHrbD9AV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FS5pm7ctk9zgPfbyHrbD9AV.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/S5pm7ctk9zgPfbyHrbD9AV.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6ac6525fc5eed6450e2edaf8f5b96777db6293e78fc070eebee594338d610e9d","result":{"confidence_high":90.72446417808115,"confidence_low":80.99270753909056,"sample_count":null,"score":85.85858585858585,"score_display":"85.9","source_stated_rank":82},"run_fingerprint":"9db6cfccb280631ac66680d3a0ba9d2b061172057879e8f260908fd985992281","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:17:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:17:31.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8fa9acb0d6a59b15f37d1e916625e60ca51497ae334954498e430e423337edaf","metric_details":{"best_score_across_scorers":"0.8333333333333334","model_release_date":"2026-06-09","stderr":2.6552207828215257},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"d23hYvSNGFo7rniLFyBWiu","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fd23hYvSNGFo7rniLFyBWiu.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/d23hYvSNGFo7rniLFyBWiu.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"ea396a59d7b3d6c604356c7df63068ec22fe13aaa4800fba9e1777342b76b634","result":{"confidence_high":88.53756606766353,"confidence_low":78.12910059900315,"sample_count":null,"score":83.33333333333334,"score_display":"83.3","source_stated_rank":104},"run_fingerprint":"d47327cf411b3c16a6f221b673a01e5b4d0f46d28ff2f744a6a51afb5f5725a0","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:17:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:17:37.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fe89a1f7e7695d2bf63ac03867838242e0193c560e4ecdbfe066cf7bb80487c2","metric_details":{"best_score_across_scorers":"0.7878787878787878","model_release_date":"2026-06-09","stderr":2.9126522834586774},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"U5PsUdX2ak4kJ2hgnQkWnX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FU5PsUdX2ak4kJ2hgnQkWnX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/U5PsUdX2ak4kJ2hgnQkWnX.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"c042c4d7dabd3778eea21a9a8b945d79254f85f4916987b484cf4ab9e153cef9","result":{"confidence_high":84.49667726345778,"confidence_low":73.07908031229978,"sample_count":null,"score":78.78787878787878,"score_display":"78.8","source_stated_rank":133},"run_fingerprint":"75f84dd8f1bec605f7530bb53f41ad6665f0874fc70ef706ae913287fbaec103","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":118,"at":"2026-06-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-09","status":"stale"},"measurement_id":"a418c99af63ff4788fa602206469e5f6e1656c74b2aecb36e72c1f26ff31f04e","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.082276,"date_is_proxy":false,"latency_seconds":24.998,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-fable-5","source_locator":"Results · Overall accuracy","stderr":0.278},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"dafa13f212f68efd79958fbbf805d11b085e57c2c0b24953619bc2b88ec6d4cb","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"70c11b415e69b606905cd5b5749db6b27c9612a8a2b4f9ab73a16a3d12a6ca5c","result":{"confidence_high":92.04688,"confidence_low":90.95711999999999,"sample_count":null,"score":91.502,"score_display":"91.5","source_stated_rank":3},"run_fingerprint":"68bf3c12d75307411cf92c7b4e39654a21d31b0b3c9c6083582684b72927e0a5","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T20:03:17.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T20:03:17.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5b90baac6c19bc8485ef379c5374d9e3efc24a31ad015931d2fbf359cd7c2037","metric_details":{"best_score_across_scorers":"0.9972222222222222","model_release_date":"2026-06-09","stderr":0.2777777777777778},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"eiLEpRUvK48wUKDh4sYkzV","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9c3130e76aa52a956a1a814788fd50c9494371f314219961e8c31b3012597ef6","result":{"confidence_high":100.0,"confidence_low":99.17777777777778,"sample_count":45,"score":99.72222222222223,"score_display":"99.7","source_stated_rank":12},"run_fingerprint":"238949feeba4fafcafe56634248cbe1a18c52e455d5e3d104ec1ac13bf986d47","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:18:19.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:18:19.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"790940a4763cb220d8f937d3cb1aa68cb249ed6a0f8a29477dcfc99f470a7ded","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-06-09","stderr":0.0},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hwrnvtEbDAocFtjcsHqsQh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhwrnvtEbDAocFtjcsHqsQh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/hwrnvtEbDAocFtjcsHqsQh.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7145e997ebc75b1cc68e848e4768dc88e8f129e3867658700aad005d7beea41a","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"3e26cbcd0b416f4692b20344c207b39b06358c743b3202f94f82edfe4f3c2502","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:18:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:18:24.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a26afb605a9b081b9a7bb5bbd94eeaf13dea2d6c0e7f2bc3d2f8e7b36f16c0ee","metric_details":{"best_score_across_scorers":"0.9777777777777777","model_release_date":"2026-06-09","stderr":2.2222222222222223},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6E989Xxeuw925T7cCV3XGB","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6E989Xxeuw925T7cCV3XGB.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/6E989Xxeuw925T7cCV3XGB.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8d481ca9ede727366cebd7f0442d6747fa660dc7e9c01606bd8ef73025ad7be8","result":{"confidence_high":100.0,"confidence_low":93.42222222222222,"sample_count":45,"score":97.77777777777777,"score_display":"97.8","source_stated_rank":25},"run_fingerprint":"867ed013b231b6a12068c2fcf8b8318f57a11e180cef20a0378ebdb794341cee","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":117,"at":"2026-06-09T18:12:35.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-09T18:12:35.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"297fa68d2c27792a57329670e2a7d5b9386e15a2ba86bd4c802cc744caf5f476","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.8701754385964913","leaderboard_private_problem_count":285,"model_release_date":"2026-06-09","public_example_count":10,"stderr":1.9944477920133024},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"E28uZiJvrPgKtawbJpoZQ8","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FE28uZiJvrPgKtawbJpoZQ8.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/E28uZiJvrPgKtawbJpoZQ8.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Fable 5 (max)","thinking":null,"upstream_model_id":"claude-fable-5_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"adaccfbd9ea3dcb2d9587a3303b17523448277ed37e5f06e49625bb955229994","result":{"confidence_high":90.9266615319952,"confidence_low":83.10842618730305,"sample_count":285,"score":87.01754385964912,"score_display":"87.0","source_stated_rank":9},"run_fingerprint":"b497a8684b1d27fcec17048e28d5eb09f57ca633ee84c367485f47fa14ece478","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dff536be94041e7f79483ccfea9fadf0e734085fdf363592327944e5cf9bd39c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":5.449076000000001,"model_release_date":"2026-06-09T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e024ca81ea8330bd940276ccf8b5472630fa8413936da5e779560a9ef9b5e720","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":89.16666666666667,"score_display":"89","source_stated_rank":18},"run_fingerprint":"314b3d3737020792e4123cf5fe55d25159dfd70830c8f5e349f8a372cf908284","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"275dccf7cddee1e8b2747cbccb35831961e4b8bbc0764c36e4428f57f5b80b8e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":3.0043564166666665,"model_release_date":"2026-06-09T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"43d8f76c04fe425755bbe1c41b4874520240d636e5c9eda322142af13ea62f03","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.33333333333333,"score_display":"88","source_stated_rank":21},"run_fingerprint":"ca68ed63b8589832656d412a8ad5e95fa677891a8241bfed4c8a84b858a5c384","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d52a4bf6c494a0166f10bbfd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"70d94ce915f52c971a610bcff83a4594833f50321dc589a1b4f65515c832fae9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":2.4358764166666664,"model_release_date":"2026-06-09T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"760f4636455a3837010955faa337be53115636257c2a2ce9d27744e16227d810","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.5,"score_display":"88","source_stated_rank":23},"run_fingerprint":"ffb82c0eb69a4b8e00f00e0776e4b9ae6d03a40e865ae1e9e2d1c1d2a9d995c8","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"af9d0458b8489a8b3ccb04dbee18524681b2d38cb95d1799cea1bd15ed32e4cd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.9269780833333334,"model_release_date":"2026-06-09T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9ab9d53cca00cb6804a30a90dd552b2ccd4d2af9541ada6b9a81ce9cf4b5a6cd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.5,"score_display":"82","source_stated_rank":38},"run_fingerprint":"790894d7e00295d370b1a680ce034310fbda9ae916583220e3beb63402978be3","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94c2d5a744c10e4a184d9ff3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5f5ced67abea63c3cad6b18c54e3bc847abba6eb0b36fe5ac2f40748eb63d057","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.4489280833333338,"model_release_date":"2026-06-09T00:00:00.000Z","results_url":"https://arcprize.org/results/anthropic-claude-fable-5"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-claude-fable-5","model_type":"CoT","upstream_model_id":"anthropic-claude-fable-5-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"76289d15b79ccb0a0d32bcb4ef8c9ae603880a180f2fb1f85d4e7aaa49be96b2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.80555555555554,"score_display":"77","source_stated_rank":44},"run_fingerprint":"4b1277e6cacc53c7775fc88077af7ca8dc8749d07af426a48798f9b31c01078e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a176a27d8cf897a88360d0e6d5d5ee38b3b82349de1566149cc41aaa3a91bcda","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (Max)","source_effort":null,"source_model_version":"claude-fable-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e46f8086b4c01af08b14a45046e068889609af6194dd0cfb3e25d61a3ca07079","result":{"confidence_high":91.973403,"confidence_low":85.532852,"sample_count":360,"score":89.16666666666667,"score_display":"89.2","source_stated_rank":18},"run_fingerprint":"1ba31ce7aad20728db395a77746edc1dbcf74b32e08cafa87465c70abf2addab","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4aa1374df26276f3a187cb17cb749c52319c28346938d9946f382c01df9d389d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (XHigh)","source_effort":null,"source_model_version":"claude-fable-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"adb0165debfa924ddf6e27aed2de9390ea163c5e19b38492ec060705761f5893","result":{"confidence_high":91.251976,"confidence_low":84.60521,"sample_count":360,"score":88.33333333333333,"score_display":"88.3","source_stated_rank":20},"run_fingerprint":"ca0f2443539b02343ae6dcc1e574f3fc66c878a53294c5a43e1cec657c4d0b71","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d52a4bf6c494a0166f10bbfd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fc7e6b0dd39ba3b74093be5a70f98674dfb49a1b68c10eccb7ec503ce2ab2a87","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (High)","source_effort":null,"source_model_version":"claude-fable-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ef892ac486b6edd7075ef3f3545dc7547cefccfd8057734ca5a1027dfa0e48cf","result":{"confidence_high":90.525325,"confidence_low":83.682792,"sample_count":360,"score":87.5,"score_display":"87.5","source_stated_rank":23},"run_fingerprint":"5ec97bc900e75c8cab2449282aefea8f884da3f1a4c1032ffaf28c9aa0c098a4","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ec30e40bef7d3b276eaaa5ef73b6a71e9abcdc272b2ccb1869c1b17c9313e574","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (Medium)","source_effort":null,"source_model_version":"claude-fable-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c29e0e8317804e89f48f90576af55421fdb1f6493e796110a87cedd92b1834eb","result":{"confidence_high":86.076226,"confidence_low":78.237476,"sample_count":360,"score":82.5,"score_display":"82.5","source_stated_rank":40},"run_fingerprint":"ee237b7909c0ae304f3f124f475be4de9c79d8ec422d29bb154bb0cf0619ef4a","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94c2d5a744c10e4a184d9ff3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"11cd7379d48e29df2d80035810875a02c4738976fe8752995f4ceb0604964447","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (Low)","source_effort":null,"source_model_version":"claude-fable-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"79ed5ccafd50fcb696564aa4f50574e34fdc6a7128b2e85f75c58013d31cada3","result":{"confidence_high":80.868744,"confidence_low":72.176318,"sample_count":360,"score":76.80555555555554,"score_display":"76.8","source_stated_rank":45},"run_fingerprint":"ff5ce2811a10ca4855d72d5e767a0fa30dbbcf6d249cd1be851deca6807b4a54","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":117,"at":"2026-06-09T18:12:35.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-09T18:12:35.000Z","first_observed_at":"2026-09-07T16:34:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"6861b50031789e4d3745dd77ede34487695d8cd89251f05d24277dd3a3a97146","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.902","leaderboard_private_problem_count":41,"model_release_date":"2026-06-09","public_example_count":2,"stderr":4.6},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"ReDAtJGGfRtrEGGWf4Zbsd","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FReDAtJGGfRtrEGGWf4Zbsd.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ReDAtJGGfRtrEGGWf4Zbsd.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Fable 5 (max)","thinking":null,"upstream_model_id":"claude-fable-5_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"d9161e1a1af891cceac563616939e3f46ecce6fba54d98fa4deab1da7477b514","result":{"confidence_high":99.21600000000001,"confidence_low":81.184,"sample_count":41,"score":90.2,"score_display":"90.2","source_stated_rank":6},"run_fingerprint":"4f1e552660bcbc523c2f64a252f5860961bfe2ce3d7c6139e9ae56c2fec9a5c8","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":118,"at":"2026-06-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-06-09","status":"stale"},"measurement_id":"0be6dba24d5c72efe3d6312fbec03870aa4afb711e2b0a3d0f6af0756eb28f5c","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.16064,"date_is_proxy":false,"latency_seconds":61.44,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-fable-5","source_locator":"Results · Overall accuracy","stderr":0.743},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"cd9093c8a7ab43daf5d8eb088832dc7bd5a08bc81aa5d7d5798ab4fdbc98d8fe","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"44132b7cb0502da0cdefe1e1c9357ecd1b91d8261782651aa6032a9080cfd413","result":{"confidence_high":90.76228,"confidence_low":87.84971999999999,"sample_count":null,"score":89.306,"score_display":"89.3","source_stated_rank":3},"run_fingerprint":"079fe09799d41ec123b6bf7f919950142844b8dbcd893ad6539b039f999a9a9f","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ac56d13734667d17b018bc8313a71df501570e300c6911d1eee2bec2ecbd5ede","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)","source_effort":null,"source_model_version":"claude-fable-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"d0da536352283b5c44f19bf761a14e3fd258e8a5521964e9d47518499c1e8c26","result":{"confidence_high":39.965253,"confidence_low":19.37745,"sample_count":71,"score":28.571428570000002,"score_display":"28.6","source_stated_rank":24},"run_fingerprint":"b13e5c256d2c27a9c48a47f0cc88b0f77e7f89bf8bbe9cf36e536422dc382e3b","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"45d532473ef3dfc4c3b5ebdf9a80200509dd88e4bae15b2a319a9c9f3183fd2a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · critpt"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5","upstream_model_label":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"4321d6acc6abfdae97c5ed803648924165747bc8b7ffbc9b9eb203d02a56e7b7","result":{"confidence_high":40.050418,"confidence_low":19.322077,"sample_count":70,"score":28.571429,"score_display":"28.6","source_stated_rank":null},"run_fingerprint":"f4032edccccba4f202a1d96907ff7d2deb1f3decfd3b18f31c6982b8dd1a48a3","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_a31432cd65f025d42eaa2c1e","provider_config":{"source_label":"adaptive"},"provider_mode_key":"Adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b2c1a8915fa59c5a470f6d417505027e959c865dfcf7a45ccef6ef544c0cf881","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-fable-5_max","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"cace3a02d8f4471dfb6c4003910470ea84db930ebd09f70883858ebf1beabdd0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.9,"score_display":"81.9","source_stated_rank":1},"run_fingerprint":"360ab2a12e9f0e7fd9b55fc41877e16fa2f7bd033f89e3c439a771627fa7ad5d","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":117,"at":"2026-06-09T20:20:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-09T20:20:14.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f93544850b94ba9880f04bbaefc72e055fa501ddb46868291c3af06bd805b5e1","metric_details":{"best_score_across_scorers":"0.41","model_release_date":"2026-06-09","stderr":4.943110704237103},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bAR85KQmEATQP8j764jpBA","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f401f150dc6acc98f1a34c486ecf5753266f20c0a0f8494fff1f222cc1830102","result":{"confidence_high":50.68849698030472,"confidence_low":31.31150301969528,"sample_count":100,"score":41.0,"score_display":"41.0","source_stated_rank":22},"run_fingerprint":"42b03880a7aa0c6adc0ebd66969177182dc0a5e994d7b8f784ce4590b4252319","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:18:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:18:00.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"830b041a7d72745b3b99801ad511b6f82d7ad3110f91a57c6e29d2a7b0c32875","metric_details":{"best_score_across_scorers":"0.41","model_release_date":"2026-06-09","stderr":4.943110704237103},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"SoCkYbcxzkYHXqcqnEQwFN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSoCkYbcxzkYHXqcqnEQwFN.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/SoCkYbcxzkYHXqcqnEQwFN.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"178700cdb7f21f027fb406ee38ef3a4f8b8622103fc49c49fc7dac6f782aaae3","result":{"confidence_high":50.68849698030472,"confidence_low":31.31150301969528,"sample_count":100,"score":41.0,"score_display":"41.0","source_stated_rank":22},"run_fingerprint":"e7122fa9b408cb73bf1b95e4993e15116064cff2dd85289ee9fcb5646998c9b2","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:18:04.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:18:04.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a1bc594512b2ed948152c272fe21a71f93f76f5ec180925715f42bef42e3c19c","metric_details":{"best_score_across_scorers":"0.28","model_release_date":"2026-06-09","stderr":4.512608598542129},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"aftzew4ra4iax5D4R4vvfF","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Faftzew4ra4iax5D4R4vvfF.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/aftzew4ra4iax5D4R4vvfF.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c43c609212c16395425774f3848adf35d84e59b4fd6820d86984f6fc5a0fbf61","result":{"confidence_high":36.844712853142575,"confidence_low":19.155287146857432,"sample_count":100,"score":28.000000000000004,"score_display":"28.0","source_stated_rank":57},"run_fingerprint":"920b8410e30c8b3a9ed6cc550fc2dac93d09eb7b0026d077be5b7b41f6d68c48","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d42184989e553211880c44fc6a9c8c9d788ed8eadd2663d081bf6e90aabad06f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (Max)","source_effort":null,"source_model_version":"claude-fable-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b0414f41aaf342e7ff1cd853c181942869f80751290a1146b532d5a39da93666","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":98.5,"score_display":"98.5","source_stated_rank":1},"run_fingerprint":"d1a0c65b0fde14bc98009586176faf67845191fc1c2a866da4efbecb439e4560","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c9f45f75b637b6bc024c5074579ebffabeef6c7e12fbcc5c2cc241f56f1a1916","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (XHigh)","source_effort":null,"source_model_version":"claude-fable-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3687537b5f1198c6ee462053643edeea1b890a83e405e2473c382ea8ce46d556","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":98.5,"score_display":"98.5","source_stated_rank":1},"run_fingerprint":"0f94e10412cea3c6bfcfcb99f50b70bb4e7c08ccf6566dbfcc0287e9e2dc679d","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d52a4bf6c494a0166f10bbfd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"60d6a1c8d5a8c9599661549f6877f775f35b96bb41bf35ea56d6b5d04e66b41e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (High)","source_effort":null,"source_model_version":"claude-fable-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6732a9f7a4d0cd91f1dd4d6aea2c2d185d0c13372aa1437abdb7ca7ee4374a80","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.5,"score_display":"95.5","source_stated_rank":29},"run_fingerprint":"d7463d7526ad9aba289f97efc2694f3e77860b4097f2e857b6979e464422032a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b9e2fcd321be3b6ef8aeb1e8418004044f90db94fe0cb2cf03d1dcb30f2a9a19","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (Medium)","source_effort":null,"source_model_version":"claude-fable-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"315f616aebc03ffd6417099d500a3418638fbb4d229a53ed01faee0e5aa00e40","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.5,"score_display":"92.5","source_stated_rank":54},"run_fingerprint":"7e06395ef72ccdaeb5590db9e788d84db1bfbbb42e84ac8b6bb3accc5c45f6e2","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94c2d5a744c10e4a184d9ff3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c68c677456ba60bb3be487e21c283a88379b1f771346cc138eae98afab534bf7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (Low)","source_effort":null,"source_model_version":"claude-fable-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a0bdbeeff3a1202e38666387939bffcadbf6926b2acf7b8edb06a8070e951817","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.5,"score_display":"90.5","source_stated_rank":70},"run_fingerprint":"009d58c02e84eea79a915063dd8eebb9143ce1912a5e502ae226951eb618c3e4","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":118,"at":"2026-06-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-09","status":"stale"},"measurement_id":"295a1cfc0bb98f5c8999d1d125dec8312a0f2c547f53fb1b4e898dfce6421070","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run; vendor scores using a different agent or tool setup remain separate supplemental runs.","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","cost_per_test_usd":2.046736,"date_is_proxy":false,"latency_seconds":356.182,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-fable-5","source_locator":"Results · Overall accuracy","stderr":0.976},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"dafa13f212f68efd79958fbbf805d11b085e57c2c0b24953619bc2b88ec6d4cb","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"e8319450934ed866f961a782d21c306d711dd33e776816aab61d7e8c2c01f71a","result":{"confidence_high":96.91296,"confidence_low":93.08704,"sample_count":500,"score":95.0,"score_display":"95.0","source_stated_rank":7},"run_fingerprint":"2bef2f98ede0fe83c443f48b5ccfe0fdd3d87d4b9456db3183ef03fd11b18f5f","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fba1a8e8e98f75f3a3063fd0bdbd751a2b81a9e024ed34400ad1a5635f53d733","metric_details":{"model_release_date":"2026-06-09","pass_at_5":78.37837837837837,"potential_contamination":true,"sem":1.4130078505728048},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"contamination-risk","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"Fable 5 [high]__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"cfac8de7b5efb6b9dcee1a89f3f6693f4a7a3571e3f28a7533a9fdc2276816c0","result":{"confidence_high":67.2739998916272,"confidence_low":61.7350091173818,"sample_count":null,"score":64.5045045045045,"score_display":"64.50","source_stated_rank":1},"run_fingerprint":"c819e155a7400554ba965b00d4cfefdcaa0893cc6b18d19b1bfa23360fd695ff","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"contamination-risk","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"5b1d897a7c8933b8bae3d1c20b81de5d138d82c98396909cae93ca1da4ff362b","metric_details":{"ci_attempted":436,"ci_half_points":4.033474250807936,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":304,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":88.4288990825688,"mean_cache_tokens":12388674.334862385,"mean_cost_usd":21.634702092592594,"mean_duration_seconds":2092.0106240022938,"mean_input_tokens":12638832.12614679,"mean_output_tokens":118592.77293577982,"median_agent_steps":79.0,"median_cost_usd":19.23457225,"median_duration_seconds":1895.921464,"median_input_tokens":9769437.5,"median_output_tokens":115631.0,"median_output_tokens_to_pass":115551.0,"median_peak_context_tokens":201598.0,"n_attempted":436,"n_passed":304,"n_tasks_attempted":113,"n_tasks_passed_any":95,"pass_at_4_points":84.070796460177,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_fable_5_max","source_model_version":"claude-fable-5","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"797587c900d28819d65b1a843e000619b9085a9e51651fc1d82f041d9c7727b9","result":{"confidence_high":73.75824489300977,"confidence_low":65.6912963913939,"sample_count":436,"score":69.72477064220183,"score_display":"69.7","source_stated_rank":13},"run_fingerprint":"be6c80ea3b852479be8ca41f35808eaea58ee510f26dedc16d021569f3c6fdea","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"0425346420180db609e5fd6e45b9f2cc71bf41f6ec74be4800cddc94f54dd278","metric_details":{"ci_attempted":452,"ci_half_points":3.244917575471319,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":316,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":68.40044247787611,"mean_cache_tokens":7159496.181415929,"mean_cost_usd":13.414521495535714,"mean_duration_seconds":1411.5830643938054,"mean_input_tokens":7329160.601769911,"mean_output_tokens":80352.1703539823,"median_agent_steps":61.5,"median_cost_usd":11.341082999999998,"median_duration_seconds":1231.553523,"median_input_tokens":5337563.5,"median_output_tokens":76035.5,"median_output_tokens_to_pass":75232.0,"median_peak_context_tokens":141254.5,"n_attempted":452,"n_passed":316,"n_tasks_attempted":113,"n_tasks_passed_any":100,"pass_at_4_points":88.49557522123894,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_fable_5_xhigh","source_model_version":"claude-fable-5","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"416cdfc9d8dc110b0c6e65db10f07154f442748319533c88123c0dfdbb3483c9","result":{"confidence_high":73.15642200025007,"confidence_low":66.66658684930744,"sample_count":452,"score":69.91150442477876,"score_display":"69.9","source_stated_rank":12},"run_fingerprint":"fb3161f2888af8a8ae07e37d171e208fd6783382cf0d603b9e325c101fa5eaea","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d52a4bf6c494a0166f10bbfd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"591b9f9163f72ad2310f04c1e619b127c91a4bcdd81eaf482f6699822f9faecb","metric_details":{"ci_attempted":430,"ci_half_points":1.1208427174303877,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":295,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":58.73720930232558,"mean_cache_tokens":4711714.3162790695,"mean_cost_usd":9.177638214788733,"mean_duration_seconds":1062.3037221116278,"mean_input_tokens":4833178.365116279,"mean_output_tokens":57287.06976744186,"median_agent_steps":49.0,"median_cost_usd":7.322686000000002,"median_duration_seconds":902.496624,"median_input_tokens":3075198.5,"median_output_tokens":52623.5,"median_output_tokens_to_pass":53715.0,"median_peak_context_tokens":99524.0,"n_attempted":430,"n_passed":295,"n_tasks_attempted":113,"n_tasks_passed_any":98,"pass_at_4_points":86.72566371681415,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_fable_5_high","source_model_version":"claude-fable-5","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"7d5246e1dac165e33b8aa0ff7a93612fe0f2562169a77ed3e10b331661ef1321","result":{"confidence_high":69.7254938802211,"confidence_low":67.48380844536031,"sample_count":430,"score":68.6046511627907,"score_display":"68.6","source_stated_rank":18},"run_fingerprint":"c6c76b15c715505da75d54870fb4178169af016d3d1c34289d9f7bd98017140a","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"27fdf950425dc0d7f6c5971fa97749d9e0ddef5cf8b2de2277bae836c4c3fb55","metric_details":{"ci_attempted":436,"ci_half_points":4.421928879238165,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":285,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":48.36697247706422,"mean_cache_tokens":2848228.9174311925,"mean_cost_usd":6.088186528935186,"mean_duration_seconds":810.8967150733945,"mean_input_tokens":2942251.607798165,"mean_output_tokens":40201.35321100918,"median_agent_steps":41.0,"median_cost_usd":4.62221125,"median_duration_seconds":639.0384615,"median_input_tokens":1732620.5,"median_output_tokens":35970.5,"median_output_tokens_to_pass":35214.0,"median_peak_context_tokens":71989.5,"n_attempted":436,"n_passed":285,"n_tasks_attempted":113,"n_tasks_passed_any":94,"pass_at_4_points":83.1858407079646,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_fable_5_medium","source_model_version":"claude-fable-5","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"4e56608230543f89161a9c89f1c96efcd2b41563f8ef40e0261359d5a0cea873","result":{"confidence_high":69.78890135630239,"confidence_low":60.94504359782606,"sample_count":436,"score":65.36697247706422,"score_display":"65.4","source_stated_rank":26},"run_fingerprint":"e672c31ff633d1cced01c8d3d242e12b2bdb54319ea25f9d9c2a0091a4165ef7","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94c2d5a744c10e4a184d9ff3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"c50c8cb858e4ecd12c31745eee2ef5db3b766d2cf2c5b67ab3c6a22534d79bab","metric_details":{"ci_attempted":433,"ci_half_points":2.7940054451297294,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":258,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":37.79907621247113,"mean_cache_tokens":1624197.8868360277,"mean_cost_usd":3.757873125874126,"mean_duration_seconds":634.7057336951501,"mean_input_tokens":1691250.452655889,"mean_output_tokens":25242.833718244805,"median_agent_steps":29.0,"median_cost_usd":2.7559204999999998,"median_duration_seconds":397.628238,"median_input_tokens":815862.0,"median_output_tokens":22339.0,"median_output_tokens_to_pass":21899.5,"median_peak_context_tokens":48670.0,"n_attempted":433,"n_passed":258,"n_tasks_attempted":113,"n_tasks_passed_any":92,"pass_at_4_points":81.41592920353983,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_fable_5_low","source_model_version":"claude-fable-5","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"40c008f2549976056ff8468f07e8378145001491ea5fbd1cc5768ef2a879e676","result":{"confidence_high":62.37830105713898,"confidence_low":56.79029016687951,"sample_count":433,"score":59.58429561200924,"score_display":"59.6","source_stated_rank":34},"run_fingerprint":"0642861fa2d6820cdc81e388166d4532fb6e112bec8b780f23cabbc694316006","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"72c36885d9da8776ab23597f25af845b2d8476cc31158e89ffd94223e6cf77bd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"88.4288990825688","mean_cost_usd":"21.634702092592594","mean_output_tokens":"118592.77293577982","model_release_date":"2026-06-09","notes":null,"pass_4":"0.84070796460177","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-fable-5 (max)","source_effort":"max","source_model_version":"claude-fable-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"c22e5f30505a98a886f4a05fab14e816511f6f4afe2a0c5ec3a5eea10c43f834","result":{"confidence_high":73.75824489300977,"confidence_low":65.69129639139389,"sample_count":113,"score":69.72477064220183,"score_display":"69.7","source_stated_rank":13},"run_fingerprint":"208e978a4146eb5291d33a5a85a01d38e2f189bced182ee538eae051e2f16768","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"eb7840107c8e03cf703075709621dd3911d92dfeae6913e644c221eee4ffe580","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"68.40044247787611","mean_cost_usd":"13.414521495535714","mean_output_tokens":"80352.1703539823","model_release_date":"2026-06-09","notes":null,"pass_4":"0.8849557522123894","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-fable-5 (xhigh)","source_effort":"xhigh","source_model_version":"claude-fable-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"7d0a0f0319237c72650511a28b1a670af6f1fe21e0f50d1ac03d8d078d405646","result":{"confidence_high":73.15642200025007,"confidence_low":66.66658684930744,"sample_count":113,"score":69.91150442477876,"score_display":"69.9","source_stated_rank":12},"run_fingerprint":"6610e82bbddc23e5e7237f927369609a9cfcd75180f0e5986ad1f6dd2b5a86e7","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d52a4bf6c494a0166f10bbfd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f4ab463e594ebdef6bf4dbfc29b95342b728ea94081fdfa5568fee704f4be04c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"58.73720930232558","mean_cost_usd":"9.177638214788733","mean_output_tokens":"57287.06976744186","model_release_date":"2026-06-09","notes":null,"pass_4":"0.8672566371681416","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-fable-5 (high)","source_effort":"high","source_model_version":"claude-fable-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"23e38c80d4e422c5f0baf6881f11b471ccc06c65c0d9cd092d1c1adbc6215b47","result":{"confidence_high":69.7254938802211,"confidence_low":67.48380844536031,"sample_count":113,"score":68.6046511627907,"score_display":"68.6","source_stated_rank":18},"run_fingerprint":"cf30bc7f538607a47fdaa6cf9fc0da5f44595b404722bbed7d5f89547768bd66","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"06d20fff546c270498d8d4b32f965d0d3e7accf21bcfb9adc27c92285a16d9cf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"48.36697247706422","mean_cost_usd":"6.088186528935186","mean_output_tokens":"40201.35321100918","model_release_date":"2026-06-09","notes":null,"pass_4":"0.831858407079646","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-fable-5 (medium)","source_effort":"medium","source_model_version":"claude-fable-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"564fafe506d9212a735b795b9c50e3ee1f38429f8fa7c6dc21d295f5cf1483db","result":{"confidence_high":69.78890135630239,"confidence_low":60.94504359782606,"sample_count":113,"score":65.36697247706422,"score_display":"65.4","source_stated_rank":26},"run_fingerprint":"129755c93ae87eaf5a37a50eb62be868cb057430aca617bf68dbdd31ff068488","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94c2d5a744c10e4a184d9ff3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bdfcde4dc369e8aa41f226d3daf55be23872de0f688c03a400a7ad71d6f9f47a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"37.79907621247113","mean_cost_usd":"3.757873125874126","mean_output_tokens":"25242.833718244805","model_release_date":"2026-06-09","notes":null,"pass_4":"0.8141592920353984","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-fable-5 (low)","source_effort":"low","source_model_version":"claude-fable-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"6d4991fcf060f32030f5f16095a1b8533ef9e6d18fee141f2333f6721fc62b2e","result":{"confidence_high":62.37830105713897,"confidence_low":56.790290166879515,"sample_count":113,"score":59.58429561200924,"score_display":"59.6","source_stated_rank":33},"run_fingerprint":"012dd11e27f9ef53478117b780866fc5209df4361c25d0a92b1cf2941146b63c","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_e361abe5266a3ddfdcef017b","split_or_window":"openai-provider-comparison","unit":"percent","version":"DeepSWE v1.1 · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":118,"at":"2026-06-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"9d7032d6029e44328832c4e2842061c69844f4b6d02f203deafb049b8432b227","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · DeepSWE v1.1"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value for DeepSWE v1.1; competitor rows are secondary re-reports; exact scaffold, task coverage and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-06-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"coding-agent tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"41cbc1ad0561acbe4168b68f451e029973e8421e04cd246efd8d9eba3e89bb11","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.7,"score_display":"69.7","source_stated_rank":null},"run_fingerprint":"0f8d2c9634c1f19744a41223680a9655a870be925225136d403d537e1642db2a","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · DeepSWE v1.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_438290606be78c73935e8816","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_7664765571d731e0f6ff7adb","split_or_window":"openai-provider-comparison","unit":"percent","version":"SWE-bench Pro · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":118,"at":"2026-06-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"e138b2447fde55a5b0b4001d5636e63b3825ead05f6596238d03f3c657a97f22","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · SWE-Bench Pro"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; OpenAI rows are provider results, while competitor rows are explicitly retained as secondary re-reports; exact scaffold, task revision and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-06-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"8430bb63f8c0e3e5b3beb4c5c22fb8da483dcc01b11915f686bd0d5f4ed3fae9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.0,"score_display":"80.0","source_stated_rank":null},"run_fingerprint":"8e57202dd658cdb221343d0871b603d4c9f7e3296d0fe0700a04d2a476a04d3f","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · SWE-Bench Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_438290606be78c73935e8816","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"fcb3d80b80daff6acdb31caac67604651329273fa66240fc3411473c5486a73e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"17.32","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-06-09","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"72","tokens_per_task":"103525","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5 Max","source_effort":"Max","source_model_version":"claude-fable-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"87897dd80a121378905de0a70465fdbeac79711bd24e15511a84f401c9e0a834","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.5,"score_display":"70.5","source_stated_rank":4},"run_fingerprint":"e04c973171a7a4717a621f5468c2eb70c292e4dc5342ee45578a05c61b123856","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"175f8d5cb285d7c19161e1458844e692386ebcff8ff6acca45bd1e7e1794d12d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"11.73","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-06-09","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"56","tokens_per_task":"64971","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5 Extra High","source_effort":"Extra High","source_model_version":"claude-fable-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"d44708a2aaa61e23768bfb5f7fc550eaa0bf135862962ebb0c8a7d5b70b0b00d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.4,"score_display":"68.4","source_stated_rank":10},"run_fingerprint":"f57c1004e40ac1935cf1349c29d74a5ba64a33364e8b4fda91df039174edf9b6","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d52a4bf6c494a0166f10bbfd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"47d98a17d79a75e2e798d5222210fb767d882300df9ca42192c077c00992a39a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"8.77","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-06-09","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"48","tokens_per_task":"43747","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5 High","source_effort":"High","source_model_version":"claude-fable-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"ac9866c78ba0af4fc8d1bf650d704a53f74f16c1dcbedba978ae94cd86950abb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.5,"score_display":"66.5","source_stated_rank":16},"run_fingerprint":"dc731cfdac66a5ac6d1a3b96d7361eddcda56d3a2a3405b1003138be550eee2e","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"750d792c10a058730a6a92b3b0f833dd7a70a1c82ce54537681a6b8876dbc9bc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"6.8","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-06-09","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"41","tokens_per_task":"30366","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5 Medium","source_effort":"Medium","source_model_version":"claude-fable-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"a7d6ad3ddfcf39e47fb1eb48ce73eb3630e656c964ec74b922c1fbe9e5728f5a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.2,"score_display":"65.2","source_stated_rank":18},"run_fingerprint":"b93d2920860fc5b6f4a588ec209c939c3e0f6fea6856e1be54f4209dc64c7ff4","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94c2d5a744c10e4a184d9ff3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"dd809b2085329131187b01c075d38b1a5ed904e8f4c5bbe852d124c7c1d04b55","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.46","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-06-09","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"31","tokens_per_task":"18182","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5 Low","source_effort":"Low","source_model_version":"claude-fable-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"715c34a0c8c065b12f4df3852e40d92007de1396867c2ddcdd44b23a023afdbf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.1,"score_display":"62.1","source_stated_rank":26},"run_fingerprint":"d2258ced60aff1741ebc806a7cb1b3a7489ce7f887ce01caee5624ab1223b3df","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e10b3c632c07a2140037839d3420dcd2e0743de27228cb85b0723d160de543e0","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5","source_effort":"xhigh","source_model_version":"claude-fable-5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"6649cc8bb042759199d52c18d26911120b5def9ce97292c82302590f034245eb","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":53.480000000000004,"score_display":"53.5","source_stated_rank":2},"run_fingerprint":"be3b15ba539cae3f586d1d8b7bfe769c09c493b39d5f365f692dcd1841aa4623","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d52a4bf6c494a0166f10bbfd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":118,"at":"2026-06-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-09","status":"stale"},"measurement_id":"e1e1f0f6e439ba391f3477e781e446e18819ebf49a176dc7573feed7c07a6cdd","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.42816,"date_is_proxy":false,"latency_seconds":118.53,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-fable-5","source_locator":"Results · Overall accuracy","stderr":0.892},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"dafa13f212f68efd79958fbbf805d11b085e57c2c0b24953619bc2b88ec6d4cb","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"bf5767ee193a3b29c1e719d69ce6d8255f5a384e59d963e53691ff02af796015","result":{"confidence_high":91.52632000000001,"confidence_low":88.02968,"sample_count":null,"score":89.778,"score_display":"89.8","source_stated_rank":2},"run_fingerprint":"7a9c66018ab52ed6bbd94a92c1754738a4d0b7e4c7c87eb35e5187cbeecbc81d","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"60442dd4e8d9b76727c70a528d4c42fb9a8598c585d46020928e85657479e454","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)","source_effort":null,"source_model_version":"claude-fable-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e0b2fcc0787cb427754dd973e5b44c18fa280fdf3ab70a7c7fd0a385185b901b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.9953703703704,"score_display":"61.0","source_stated_rank":4},"run_fingerprint":"424e32bd2fe24b0e7264ebb9a6e209baa547e167ac8036fb7a4541dacc99840e","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6f39f4eafaa98520066d8e90e21def2a52e727d6dd2c1e3459bb75c1ad84846f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · scicode"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5","upstream_model_label":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"19115ca0983f94a7d56182fc5607d891776d37fdc2d67d4246c07739a414d9a7","result":{"confidence_high":66.448651,"confidence_low":55.252618,"sample_count":288,"score":60.99537,"score_display":"61.0","source_stated_rank":null},"run_fingerprint":"398bc71ff2db17d54d7c9d949d656539cb67f41ecab36f83bc68a0e55c2e2697","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_a31432cd65f025d42eaa2c1e","provider_config":{"source_label":"adaptive"},"provider_mode_key":"Adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"537a3b2187ac1842d48067d927d0567895ea2aab1a425f662c3b5827c3afb51c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"2.108822","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-fable-5-high","source_effort":null,"source_model_version":"claude-fable-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"dc8244642f7f31390dd20058e4f48b24ed9d0984f3ccb1e5aa415514fec7f53c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2041.3,"score_display":"2041.30","source_stated_rank":7},"run_fingerprint":"1d23a2d2397bb679b31fad528e0a95d441e50a4caa037a9dd5e80a53ce53ea48","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":118,"at":"2026-06-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-06-09","status":"stale"},"measurement_id":"6aaba5a873aad56e139223aa77e2f0fc1112eca85aca57548a0076171b2b377e","metric_details":{"benchmark_page_updated":"2026-10-02","cost_per_test_usd":41.714694,"date_is_proxy":false,"latency_seconds":3711.026,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-fable-5","source_locator":"Results · Overall accuracy","stderr":2.097},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"cd9093c8a7ab43daf5d8eb088832dc7bd5a08bc81aa5d7d5798ab4fdbc98d8fe","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"611d9ba63a35afc342de568ccfb113c71147647496045807464c0cee02d94524","result":{"confidence_high":94.46212,"confidence_low":86.24188000000001,"sample_count":null,"score":90.352,"score_display":"90.4","source_stated_rank":3},"run_fingerprint":"8b1c66ada8ec0be43908ba888e9d8f1ab9385b18e60fe3940ecc7bab21278679","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fac291013adea5d05d954104830056d2398e9deb10fe8625e355353bbbb6fc84","metric_details":{"license":"Proprietary","variance":10.691833903598845},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-fable-5-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9ed34cf3db4f4fe3bc8b12611861cd2010d1bd9d2d517c3bac26ea8007e8279e","result":{"confidence_high":1631.7076745488966,"confidence_low":1618.8901490999847,"sample_count":14126,"score":1625.2989118244407,"score_display":"1625","source_stated_rank":18},"run_fingerprint":"ee9a7622ce03e23f03b37333cbf8cf765a3e3255a6b65cb1237766ddd99160e2","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e97bd885b73bc88b229d91354a767f7ae73ac9fe3b8566f4ef010195bdb073b9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-fable-5_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"f7caf39f524d8d197f009ea9062a35d8d61bba97531d27bbcbbd1b938d7d52ef","result":{"confidence_high":1635.44,"confidence_low":1620.53,"sample_count":10081,"score":1627.98,"score_display":"1627.98","source_stated_rank":13},"run_fingerprint":"bef27707b7e9c170718c7efc35af7d7502063f1bf57d036f01479d1c38eb30a5","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c1205d695f3f92b92d0843672104c1075b571703aa0cf31860ac506529c77a47","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-fable-5_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"1e832485bb134925b28acf009d768fcdb67a2ec8a16ab7ab10386e29d2c6a347","result":{"confidence_high":1634.46,"confidence_low":1618.06,"sample_count":8382,"score":1626.26,"score_display":"1626.26","source_stated_rank":14},"run_fingerprint":"2049dc9d30ea71687ba70cf1d5e11e097f529945f73cc4287d5f19cd45331a17","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5e358d68414dff7e7529bc0b","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"mirrorcode","metric":"mean_score","name":"MirrorCode","release_id":"release_07bf24088c0805f7b6df7a88","split_or_window":"15-programs-30-language-tasks","unit":"percent","version":"ML +Private 2L"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T16:12:44.227Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T16:12:44.227Z","first_observed_at":"2026-08-28T19:57:34Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fd63a518f4eaefdc9711e65fa07958f84d4b732f5d0fdbc9f9ea3e332537b197","metric_details":{"best_score_across_scorers":"0.638888888888889","model_release_date":"2026-06-09","stderr":10.404092940368063},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mirrorcode-ml-2l-claude-fable-5","log_viewer":"","logs":""},"run_count":3,"scaffold":"Epoch AI Inspect ReAct agent","tools_allowed":"shell, text editor and visible-test evaluator; no internet"},"protocol_fingerprint":"a41ce8d08c70f058547b71d087fd53b98aaffd997c3316745d4a2fe42e4ddc5b","result":{"confidence_high":84.2809110520103,"confidence_low":43.49686672576749,"sample_count":30,"score":63.88888888888889,"score_display":"63.9","source_stated_rank":3},"run_fingerprint":"220f9c819ad2cd742419f08e25599baf0c9872656bcd8e9051955f0991085539","source":{"content_hash":"3e212e97e03d0343e676ecc7ff046ee7b94d219ba7d863364634b7361dc89b4b","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://epoch.ai/benchmarks/mirrorcode","id":"epoch-mirrorcode","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · MirrorCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/mirrorcode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2985cf3b58bdcb4be6b4c71d6e372c98ae097628fe20cdad6a7a5504091f3a8b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-fable-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"b9e36ef7392f80cf55925733544effa6dfb854521664ff9ded2b243e4b70e2cb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.94,"score_display":"91.9","source_stated_rank":6},"run_fingerprint":"92341deb2d474ace3dfbb9ef2ef8c0ba6d1ee6e617998f79838053560a8f6a03","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"75eb6930dc7228357afffc0a741e9d48ca69839fb441a2ac4d64da6e5d41a48a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-fable-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"ce85fd1868be093b2c96098773db24facd03ea98256e69bacd95c0178aff5e40","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.85,"score_display":"87.8","source_stated_rank":11},"run_fingerprint":"e842fe7d2028d88adec3ba191255be8a023cf225110b1ee4e1f5c58b1084e252","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":118,"at":"2026-06-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-06-09","status":"stale"},"measurement_id":"eb9f1caeb39ef63b346e54d32230f439f3b22257274e53d09b8a1c0b27301fb3","metric_details":{"benchmark_page_updated":"2026-09-28","cost_per_test_usd":1.429025,"date_is_proxy":false,"latency_seconds":504.537,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-fable-5","source_locator":"Results · Overall accuracy","stderr":1.35},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"cd9093c8a7ab43daf5d8eb088832dc7bd5a08bc81aa5d7d5798ab4fdbc98d8fe","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-fable-5"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"f63f7da9f9a11f6aed7c2b250372a8100596ad6a6eee3506e874005c4edb6e8d","result":{"confidence_high":83.17,"confidence_low":77.878,"sample_count":null,"score":80.524,"score_display":"80.5","source_stated_rank":10},"run_fingerprint":"8723dac9e6a60171e6078cf67221ec48bdeed67622ec3f62e792c6405f672989","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8a8895d643a7e6695b68e735c920576a89f7026f648a8d44942b6ca5564ebb01","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · terminalbenchV21"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5","upstream_model_label":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"1af3db0ae3a27423c057218c953d3bf29d1005e9a51a8cbb22c2537c60b86e96","result":{"confidence_high":90.683119,"confidence_low":75.738256,"sample_count":89,"score":84.644195,"score_display":"84.6","source_stated_rank":null},"run_fingerprint":"8f3287be0f500f296cfb5edd40db225450f36766171d855febcfbc0c7517155f","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_a31432cd65f025d42eaa2c1e","provider_config":{"source_label":"adaptive"},"provider_mode_key":"Adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_6d7e85c927ba2919129973e2","split_or_window":"openai-provider-comparison","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":118,"at":"2026-06-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"2bf7b4324f64f6da266e1674682ce5a3e8bd17c9e526b4477d40cb4008e5d0df","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · Terminal-Bench 2.1"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; competitor rows are secondary re-reports; exact harness and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-06-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"terminal agent toolset"},"run_count":null,"scaffold":"comparison harness not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"5e230a71d1c25a71e8ebd24b7698cb27f830a6a422de5a306f016a81a92f9da8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.1,"score_display":"83.1","source_stated_rank":null},"run_fingerprint":"4b47685bf46663b1938adc3b94c44422749fb98343fb32bf3ea54f253d954e33","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · Terminal-Bench 2.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_438290606be78c73935e8816","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T21:39:34.661007+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T21:39:34.661007+00:00","status":"fresh"},"measurement_id":"4a3bed6427323f5a1cf334d3ded74c3caaaa3c1d033cac8072648dbcd43c4927","metric_details":{"accuracy_stderr":1.71,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:06:23.406434+00:00","total_cost_usd":6480.94,"total_tokens":3583230945},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"Anthropic","url":"https://www.anthropic.com"},"agent_url":"https://claude.com/claude-code","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"b8ed61a2-838a-495f-8d24-3ebffb9be16b","model_release_date":"2026-06-09","model_url":"https://www.anthropic.com/news/claude-fable-5-mythos-5","n_trials":370,"source_row_date":"2026-07-13"},"run_count":5,"scaffold":"Claude Code","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"3d11a4e7e9474b518de9518ea5ac6bf9a0167a6e45624d8bb01ff8cf092ddc04","result":{"confidence_high":45.393824,"confidence_low":24.280489,"sample_count":74,"score":34.05,"score_display":"34.05","source_stated_rank":3},"run_fingerprint":"5318b343d3979e9d6380768d401b794c0b627f039360f56c1f55963367e2e965","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":74,"at":"2026-07-23","basis":"evaluation_at","evaluated_at":"2026-07-23","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"607c064c7ec013aa9e3540c63fca7f8ad3bd748df498be4a1c9d178c55228f05","metric_details":{"cost":null,"pass_2":32.817869415807564,"pass_3":30.15463917525773,"pass_4":28.865979381443296},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"claude-fable-5_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"bc63906b45498a36c7f69245c165cd52ba114e01c9ead5912d9063301c0508b4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.69072164948454,"score_display":"39.7","source_stated_rank":null},"run_fingerprint":"2232d2f89ae14744087af5d8a44288503c8e607f50d89312d8dc76e717342132","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"25380421a26d9a2f7a471cd187fe1834fef8bb70f94f4202ac7c7b0d7430d3a4","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · tauBanking"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-fable-5","upstream_model_label":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"e4adb326a2e894634f4ff7a2de41020240efa24c3a32d5fc523ba0d4d01f753a","result":{"confidence_high":48.08744,"confidence_low":29.104512,"sample_count":97,"score":38.14433,"score_display":"38.1","source_stated_rank":null},"run_fingerprint":"456c888cbf658073978a21446424a83f647cf48c8f079ba2163f141b9d29a973","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_a31432cd65f025d42eaa2c1e","provider_config":{"source_label":"adaptive"},"provider_mode_key":"Adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":117,"at":"2026-06-09T17:51:31Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-09T17:51:31Z","status":"stale"},"measurement_id":"b0eeb6b6f45df385a850821bc0147ca14a651bfe70977953134226cd3f75ca95","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.25,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=Claude Fable 5; createdAt=2026-06-09T17:51:31Z","source_row_created_at":"2026-06-09T17:51:31Z","task_pass_coverage_threshold_percent":75},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"Claude Fable 5","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"99173787d4cee1df6dcd2860354b3d4bf02287e86f0a4da33ea1945ba181c16d","result":{"confidence_high":85.55,"confidence_low":81.05,"sample_count":1000,"score":83.3,"score_display":"83.3","source_stated_rank":2},"run_fingerprint":"8d7a41967f2afba7b3e80d94b3d2f58c65510c72bc1fce0ef9e936951d95bf21","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=Claude Fable 5; createdAt=2026-06-09T17:51:31Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_185ead53e002c3c1ba45c533","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a9dd0185d518b7b65ffa1bc378a738bed078f7c48c62de1c68467b71d4fb3caf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.745","mean_standard_error":"4.3","model_release_date":"2026-06-09","notes":null,"standard_error_points":530.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Fable 5","source_effort":null,"source_model_version":"claude-fable-5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"0d3f8208e7740f9aa8b98d8f6ad72f3d78c2f7d3615fc450c1529291933966ef","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":63.6,"score_display":"63.6","source_stated_rank":9},"run_fingerprint":"85a460724a24dd8c60f5126dfab1b8910621aec6de66bd06817bac0080bc0383","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_482e02f0349d00ca23df8127","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a5db089c5e46f4c6007790f6e3f6cf0751633d72543c898bd363129735d48557","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":41832},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Claude Fable 5 (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d0e61141d3a6a72f484b255b78727e531de9601e24b0fcf93c786b75660e331d","result":{"confidence_high":0.09429123031212294,"confidence_low":0.06995616515766467,"sample_count":2603067,"score":0.08212369773489381,"score_display":"0.0821","source_stated_rank":8},"run_fingerprint":"21916ed39a7c55ab8b6a16c4fcf86eade302dd994c34be6445462dce0bf3709a","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"25d530606cbe60c49320c218d92c5a6429aa76ee5279247615aeee914b6674dc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 - Max","source_effort":null,"source_model_version":"claude-fable-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"e24bc75ebde244e669d9ef0cced07b951a49d50bd35095b9e3d95253c8198b00","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4966.639999999998,"score_display":"4966.64","source_stated_rank":31},"run_fingerprint":"35c28958d7ac799f037cafbc76ee7c2ebd548c621ad6a19f550d270bc6e6e21e","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6dcbb1e7672bfc38d61731cefb8c9e06f90f034a3475512a22257cb63cd69763","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 - High","source_effort":null,"source_model_version":"claude-fable-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"97e1cf67629459cdc8914b483ee9fb4c342bcfb0c5c4185bd134c144cd88f425","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5680.258,"score_display":"5680.26","source_stated_rank":20},"run_fingerprint":"c60ec57c937afb9ce9dedbed749319321b00073a269947d67783830878855809","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7f9f1833ebf92e99cdabe7b9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"05a2cf7d0b7d613c3b9e2fa03281865d6041565bc28ab0f5ea254f52adde9b9e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 - Medium","source_effort":null,"source_model_version":"claude-fable-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"c4185c08eb81e3f443efebc7d3c9ec37361c2d04472758b3aa1372e5392fd97b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4339.812500000001,"score_display":"4339.81","source_stated_rank":35},"run_fingerprint":"ea7ef8a0db1e1e5868a4ce1ff6c6a96a9fce724214c1d1c5968d828b0bfa3dd9","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94c2d5a744c10e4a184d9ff3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3bb8784ed1078214cf1c1fd11ecb4b8a07cd33b896a84f6877a93254c7a5f799","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 - Low","source_effort":null,"source_model_version":"claude-fable-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"265ad37d5110ebc158021a28d56805c2fa138655aef6ebed506c4cd3d6e944cf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5018.519999999999,"score_display":"5018.52","source_stated_rank":29},"run_fingerprint":"a094869f429d2517a574c5387a1816f68ac610620dfaeb90cedf29c371d31514","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_27f952b225196069e3ef2804","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"50a5c709b67af88c9782fb6dc04d3e683b1066cd3bc253d10b3e39ac5d9f8a7f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-09","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Fable 5 - None","source_effort":null,"source_model_version":"claude-fable-5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"df19451b3e57e6cc7293ebfdf6c35fdbd3dabafab3d06be1bc9f61911112c212","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4529.944000000001,"score_display":"4529.94","source_stated_rank":33},"run_fingerprint":"c570696900bf28069fe793aa9081e63a130482b2cec125d40c5e5943425ae880","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_09a9c11bfaf37a9741e01afb","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9ebbfe93b99eca159813d1670548dfe7ca47bc798d5b6a6917e755eaece4ca11","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-09","reported_confidence_interval":"-21 / +21","source_locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · gdpval"},"model":{"id":"anthropic/claude-fable-5","name":"Claude Fable 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-fable-5","upstream_model_label":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"61a6707a499d807cac7232da5505cdcaeb43873a88a19f0a9af9cab2c0d23b5d","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1595.41,"score_display":"1595","source_stated_rank":null},"run_fingerprint":"df9fb71f7ddf3824f4aea4203dcdd6ddc51215183791de2d4ced8c502cbaba4d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c6c9b7c96eb814bc15191e40","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cd7072974016ad2cca468773149525d2887bcb85ac539ebc57bee1dfba0cd8ab","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-09"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.6 Sol; model release: 2026-07-09","source_accessibility":"API access","source_model_name":"GPT-5.6 Sol","source_model_release_date":"2026-07-09","source_model_versions":["gpt-5.6-sol_high","gpt-5.6-sol_low","gpt-5.6-sol_max","gpt-5.6-sol_none","gpt-5.6-sol_promax","gpt-5.6-sol_proxhigh","gpt-5.6-sol_unknown","gpt-5.6-sol_xhigh"],"source_reasoning_efforts":["off","low","high","xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cfaac24d2496046752b22be65838e4e70710b140f0f9f80e3e1d1adf4b405f50","result":{"confidence_high":165.26,"confidence_low":159.26,"sample_count":null,"score":161.8,"score_display":"161.80","source_stated_rank":8},"run_fingerprint":"36ca3c9500f83e594acd6b366337f54abea5845def2bedf857f2c793abcfac1a","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7ff85e71c65ccc9bee850b2e","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aee2c8df857fd0c34c8cb257e74f74cacf4aae8cebb440fd5100b050ed628cd6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (max) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol","upstream_model_label":"GPT-5.6 Sol (max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"d2934fe184a2996fc3afbb7c297960f16e2ea42ab485e3f97af9f94bb8c1fa66","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.061357,"score_display":"47.1","source_stated_rank":null},"run_fingerprint":"62d6c3fbf9e1ad917126a63b9ca88045ca403e33ac17b36c8e5ee3c318e6d702","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"42d5a023417a30710a774be03dab5be592dd1e4653d0918135bdf9eccbfda2b5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-xhigh","upstream_model_label":"GPT-5.6 Sol (xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"455da22778598f0f9a2001db40c2c07fe17e8410cbcdad48fa3621f1b0f4556f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.138026,"score_display":"44.1","source_stated_rank":null},"run_fingerprint":"7e003909f30cbf19e3ef0f9eb4085c0f2db213a536c48e01a69e64c25018870a","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b338206c6d3217d1e39abc0d9580556d59bee081430ba61ceb7db7bc5fb9bb20","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (high) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-high","upstream_model_label":"GPT-5.6 Sol (high)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"ef433ad1262db9319b5b6cde473d61fab5bf567c414a116c1bd3708281af7fe1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.499176,"score_display":"42.5","source_stated_rank":null},"run_fingerprint":"ff41cc092c0b250a607d0d1bf5eaffb374f65f46ed2450b292afc4aa22d03a9e","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (high) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"73a4198d44d9056968d0d26390cd0a7de6a8ee7e14bfbe7a3b0eb687ba54c912","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (medium) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-medium","upstream_model_label":"GPT-5.6 Sol (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"54f1568a4cd5bec04c53dcc8db76b0565f74ebad96e115f7c5314e5087152fcd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.484559,"score_display":"39.5","source_stated_rank":null},"run_fingerprint":"7867a4848f02cf798bbab67c6c84cf548e2a43d5701f19a0003e364d897b0ed2","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"39278c8d2f21764c74c20e603454b5b7e69b7ed88f257bda38fade5726438188","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (low) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-low","upstream_model_label":"GPT-5.6 Sol (low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"88b84d6451b5fb6d710691561ff9fbef59a71ebd07f615501355f10d701e7551","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.797939,"score_display":"33.8","source_stated_rank":null},"run_fingerprint":"6776e95aebb4fc3cfc10dfe381d9538b1f7dbfedd17437d536a497f5006a63de","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"128fc3f1ba90ee2a91f523155be004d619addabb9503a4117bf53f939cc8719d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-non-reasoning","upstream_model_label":"GPT-5.6 Sol (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"39362f0b22319416b2966051cf4603058dfa7d3d47d0ddafb720c8e86e6ad9ae","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":28.328004,"score_display":"28.3","source_stated_rank":null},"run_fingerprint":"4d618d094e1efb7c80033148564ccf4766bf09a04d811ffdbc86f9c6fc47fc9a","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a948f7792ab02ad4c3c49628057a27c959e840e8d5ad30f594dac31d74a49e12","metric_details":{"license":"Proprietary","variance":5.134008853009156},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1488.378963210484,"confidence_low":1479.4970526966556,"sample_count":36656,"score":1483.9380079535697,"score_display":"1484","source_stated_rank":20},"run_fingerprint":"e987f2206251d3a2013de8822ae8f4737006d7576539b102dcc1352903e72a9f","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"aa9f51b99344e177889b5c2ec691dcc625fafdd0cea1f94533b867300bb32cec","metric_details":{"category_scores":{"Agentic Coding":56.211999999999996,"Coding":83.941,"Data Analysis":79.84066666666666,"IF":71.846,"Language":87.68366666666667,"Mathematics":96.19775,"Reasoning":91.65375}},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.05354761904762,"score_display":"81.05","source_stated_rank":10},"run_fingerprint":"86df1a485884b3522a22b1dde08bec340d454a6e6e348b3d9760d5ea6a896449","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b94b884858efda8a0f0902ecb15c2cc0e018e05da04a6ba946157cee709499ac","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (max) · hle"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol","upstream_model_label":"GPT-5.6 Sol (max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"db96fb25810622597f236f437c027d00a5acfa3d1c917e4ea2af57ad594b4bc6","result":{"confidence_high":51.598791,"confidence_low":47.383558,"sample_count":2158,"score":49.490269,"score_display":"49.5","source_stated_rank":null},"run_fingerprint":"e5ccf8a301b169e92a710894f9860495b02cfafe02e6cfc3e8dab7a1879b3afd","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8f527cb722bd28e248c80e876e311eeeedb1da350824e16246223b460b0907bd","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · hle"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-xhigh","upstream_model_label":"GPT-5.6 Sol (xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"bbe50ffc9109cdb0e9531f4d30692029fcfc8351ad5973dab3194cec4c885430","result":{"confidence_high":49.421786,"confidence_low":45.212418,"sample_count":2158,"score":47.312326,"score_display":"47.3","source_stated_rank":null},"run_fingerprint":"31024db8ee2df49566bf7b66063be1cdf65497b0f9979310dc948b10f12e9c31","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1d67f472ffeb0246dfb26a8d244cef024ae3bc4a2330e69c3a8cd3a34b33688a","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (high) · hle"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-high","upstream_model_label":"GPT-5.6 Sol (high)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"75ceb668ff6f2fd955e0aaeb583ed74f0d7391847728be0957d1359a038f8257","result":{"confidence_high":48.122943,"confidence_low":43.920878,"sample_count":2158,"score":46.014829,"score_display":"46.0","source_stated_rank":null},"run_fingerprint":"b48ab1d5cd1ca33d2588a4ea9738d22e3f83c6316f1cd53da31cbb99be6ae06e","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (high) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2f5d890597f5b0232a47b182fa6fbd135cd82cf5125fdf60e9197fd602d47cd5","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (medium) · hle"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-medium","upstream_model_label":"GPT-5.6 Sol (medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"249108ffcf57ba547a2312496f28ecbebd48178092c8eaad665e7c5e44f6025c","result":{"confidence_high":44.310915,"confidence_low":40.146781,"sample_count":2158,"score":42.215014,"score_display":"42.2","source_stated_rank":null},"run_fingerprint":"3a0f954e55809253d3972748592ee89a62f44539dcd2cbce851136dd47113a99","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a28f8f9a4318253625fd9abf99bedd322a0496979c48b0ab6f4bf8ddd5af9363","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (low) · hle"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-low","upstream_model_label":"GPT-5.6 Sol (low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"35b28e44584660876dc16d1d2f12aedbbd9108f191a90e00ea0ddbdd9ba2d89c","result":{"confidence_high":41.466976,"confidence_low":37.347384,"sample_count":2158,"score":39.388323,"score_display":"39.4","source_stated_rank":null},"run_fingerprint":"3f29ffa136bad3487dab623a421ad362be69805863a03717a4f24124e61fed0b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"be5598ab8b7264617cee2abddda5083b24696611b574e88bf58725d3b2207bb2","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · hle"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-non-reasoning","upstream_model_label":"GPT-5.6 Sol (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2adbf4294ce5ad7b028ac55b5b58794541160d858e06921c6c06dfd44525d703","result":{"confidence_high":18.314022,"confidence_low":15.168616,"sample_count":2158,"score":16.682113,"score_display":"16.7","source_stated_rank":null},"run_fingerprint":"1ae15b788e5070aedaa98c2792b9006f85d2a663692d180024efdb1ae2570c71","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T23:44:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T23:44:30.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0add7db285a818b7686865c934756344d08e3073ecadd4b7ea4fec3dcefc3a32","metric_details":{"best_score_across_scorers":"0.697","model_release_date":"2026-07-09","stderr":1.4539683710535192},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"UnRUSuZzm2X27TUbpeMPva","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e8c442b4dfa0deb1c59f43f7f6c56b371e45a3134beb26cfbd35e2d47f70d096","result":{"confidence_high":72.54977800726489,"confidence_low":66.85022199273509,"sample_count":1000,"score":69.69999999999999,"score_display":"69.7","source_stated_rank":7},"run_fingerprint":"05a13b7c486b07d9d0d6fba9f57de1defb865a717bc9eda5463488c27115883f","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-23T03:49:58.395104Z","basis":"evaluation_at","evaluated_at":"2026-07-23T03:49:58.395104Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bfbcc7b6653c26607481645aac830ae0eb92707dfe9ce55c77d1ed6e1f544afe","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-5.6-sol","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"83e6575380420c706678f19e80b0920c897863bceb85ab69dc482702eadf923a","result":{"confidence_high":72.14559540304893,"confidence_low":66.33135850476668,"sample_count":null,"score":69.2384769539078,"score_display":"69.2","source_stated_rank":7},"run_fingerprint":"55ad3270d11d323e1f453fecc7e2914390b7e5497cd77d0bf60cdd635f5d72e6","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5d0fc6b8257f27670e5f6463","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7e6a254200e5c9518ecd2575a9967b2f00ecd072c8b91c5e112b46ca7eb15609","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.10996,"date_is_proxy":true,"latency_seconds":58.304,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.074},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":94.949,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"max","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-sol","zero_shot_accuracy":95.455},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"4c8694e02c60a23f0a8ff604b03406fd98c1d6171ac8de529f7fe4b3a0cd285d","result":{"confidence_high":97.30704,"confidence_low":93.09696,"sample_count":396,"score":95.202,"score_display":"95.20","source_stated_rank":2},"run_fingerprint":"0d741ad1fc1b5d13d0b4d98b718750b8e8a32f7d490dd507a3d3a1f00caa4af2","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f7da682f1d2c9e319a3677c8f028535eef4926e96e3de520b0e26fe74600321c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (max) · gpqa"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol","upstream_model_label":"GPT-5.6 Sol (max)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4841ad13b07dc48ffd945a8ce119fa6dbfcdc34fbade3322d4c54f8025162eaf","result":{"confidence_high":96.648379,"confidence_low":89.954185,"sample_count":198,"score":94.141414,"score_display":"94.1","source_stated_rank":null},"run_fingerprint":"afd9376df47870779d4e9c0d3649d1961b42e1bffc3d8ab0df63b2a12cef847c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (max) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"44842c918c91528c657725ebce5383baeda7636277094aad5c41278e214302e6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · gpqa"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-xhigh","upstream_model_label":"GPT-5.6 Sol (xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"be2a1116049413791972c66599226595bab47a03d1d3540f4978699b02b1df4d","result":{"confidence_high":95.894948,"confidence_low":88.725863,"sample_count":198,"score":93.131313,"score_display":"93.1","source_stated_rank":null},"run_fingerprint":"c3dac9994a4007bcf3a01bd55d4cc996a29904d12b5da17afcadb377c5016833","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c9041fce21f27e7b9a23e1185d6714907087b756527daba75c9df4f4116d2f25","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (high) · gpqa"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-high","upstream_model_label":"GPT-5.6 Sol (high)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5c8feae1c80d7b1cd43d3994ce6365652124afa579a2b28b1161dee233414639","result":{"confidence_high":95.664893,"confidence_low":88.361393,"sample_count":198,"score":92.828283,"score_display":"92.8","source_stated_rank":null},"run_fingerprint":"584c53dbf19dcdf57ea9c11d8361c9c285216adc822769fabdff70d0a84f061b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (high) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9cb2034150dde9a58bb2bab6743c669f7596ca47d7ad1f1f23b0aea7ee88ec2e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (medium) · gpqa"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-medium","upstream_model_label":"GPT-5.6 Sol (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"99ffc8bc4a5de1c4fd352aea87e041b64847994c73af7b9f39403731fdee8ed8","result":{"confidence_high":95.510587,"confidence_low":88.119349,"sample_count":198,"score":92.626263,"score_display":"92.6","source_stated_rank":null},"run_fingerprint":"31669430b3c066a3fb4915d08156c36bdaa78626640c427c5025384511e420e2","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"94613c52527acb6c5ec7800250b43d6423a62448367953baeac13b57f06f2989","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (low) · gpqa"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-low","upstream_model_label":"GPT-5.6 Sol (low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6692e49cd294bc96c97e9a7bf9f9778b8f2ab6316d760bc3a30bc7ef2246ed30","result":{"confidence_high":93.28434,"confidence_low":84.79669,"sample_count":198,"score":89.79798,"score_display":"89.8","source_stated_rank":null},"run_fingerprint":"e845fa2a6d2a6789162da515d9d240c9a6be4f0db230e8efab1664be98e6f2a5","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bf3133cc6b11d48c10d50ea08a0a59e53a368b834675ef24e9bb145f34e34958","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · gpqa"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-non-reasoning","upstream_model_label":"GPT-5.6 Sol (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"80f87580b76e14c95af3b79cacd67c633f890ea051f67fd513db5b806252d34f","result":{"confidence_high":84.085345,"confidence_low":72.790938,"sample_count":198,"score":78.989899,"score_display":"79.0","source_stated_rank":null},"run_fingerprint":"9ee2ffc5dfe4805dff35a4c463018c9e7189006cf4ca58709041677a3c7a5771","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09T02:07:13.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T02:07:13.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cbd10250701088468be88dfb30e626f25e69ce7dca2b9cb5d6c8786477d7d6e8","metric_details":{"best_score_across_scorers":"0.9349747474747475","model_release_date":"2026-07-09","stderr":1.570109070867593},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"nuKCgZReCBsXKCkNj9bK37","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"06f8267d27b71a8fef63509921c34fcde7d62f06844c5440f3f58618ce017155","result":{"confidence_high":96.57488852637523,"confidence_low":90.42006096857428,"sample_count":null,"score":93.49747474747475,"score_display":"93.5","source_stated_rank":15},"run_fingerprint":"75a42cbe70cbc88583a3c1995718dfa12b3ed33bf93b01176dbadb8fc5051170","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:46:32.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:46:32.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7e631f5473bc909e92518ea484c865e860c913cf46e5c047f60fa760e3946d31","metric_details":{"best_score_across_scorers":"0.898989898989899","model_release_date":"2026-07-09","stderr":2.1469735576055355},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6cmgkNAQZekBr6e6rVdLe9","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6cmgkNAQZekBr6e6rVdLe9.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/6cmgkNAQZekBr6e6rVdLe9.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"1de8682b0b11750b4d78eeb5d60b68be7dd945b32abb77db446cea48c13c3c9a","result":{"confidence_high":94.10705807189674,"confidence_low":85.69092172608305,"sample_count":null,"score":89.8989898989899,"score_display":"89.9","source_stated_rank":46},"run_fingerprint":"f2b6b402d0b83cd980b6ded022a25a9f242996369de9f06fc2bc41a19007ee37","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T01:00:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T01:00:09.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"465e8d8e6d50e36e2ec38f9fc5fda24da7aa585718ecf6b1d6e7a63f5f6d4401","metric_details":{"best_score_across_scorers":"0.8282828282828283","model_release_date":"2026-07-09","stderr":2.6869716187429873},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"DDfYPyKQW7vgDtGgkwcKSi","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDDfYPyKQW7vgDtGgkwcKSi.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/DDfYPyKQW7vgDtGgkwcKSi.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6dbf7b0b3d4ee2ad785be6ea0ee7226cc645f2561117b6448109f21846efdc17","result":{"confidence_high":88.09474720101908,"confidence_low":77.56181845554657,"sample_count":null,"score":82.82828282828282,"score_display":"82.8","source_stated_rank":111},"run_fingerprint":"a8a1e4222514ea64c3fe3931c74c3f73182d6457b923a966ac1983d07233a293","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":81,"at":"2026-07-15T19:19:22.116333Z","basis":"evaluation_at","evaluated_at":"2026-07-15T19:19:22.116333Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8b4dfb7183cbb69d4fef40ae251c208127d8f926819d365410cad652193ae5ff","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-5.6-sol","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"35d2684b376a903474dd8b949fc0dcb02b9ec62ba9fb5f8cfad3190f5b7fb145","result":{"confidence_high":96.10995513204666,"confidence_low":88.73852971643818,"sample_count":null,"score":92.42424242424242,"score_display":"92.4","source_stated_rank":4},"run_fingerprint":"5a25d29464c59dbaf1264494a972d1b7d117260441f4423e0d87453992066f45","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7dc1bd1a5a65295cdfaa024b","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f2f8eb675d5a4d14d203e84f","split_or_window":"provider-reported-protocol","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"afd3d790a57082409cc02f5bc8340472eb58cf90a9ab0da112074782d5baefe1","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Academic · GPQA Diamond"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort and tools not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"b72ce4576adc1201eec7ec882a588d449d9c2f5cda949167eb39a584f0bce769","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":94.6,"score_display":"94.6","source_stated_rank":null},"run_fingerprint":"0b28e67f4ec98f903399e267316a3c344a8d9941e1ec6eba1c725677e47cf39f","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Academic · GPQA Diamond","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f01085c2078d97f9df207606","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"93d92385e0c930fc086b869cb73ea5db14bc54968a5fae93a31a894c606dd5a5","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.032087,"date_is_proxy":true,"latency_seconds":15.654,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.308},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-sol"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7e8d5595ed4a826dee31272ae8143db00ea18e285a1901ed0507b97fd99121fb","result":{"confidence_high":89.70367999999999,"confidence_low":88.49632,"sample_count":null,"score":89.1,"score_display":"89.1","source_stated_rank":16},"run_fingerprint":"68cce91a16a7ccb4e5dac8b61a0fcb2d52968bb0935ff7fbb2bac621786d68ee","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T02:07:13.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T02:07:13.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0f89597ddaf54a4ab770a7d953a3e5a7c5af91f048ece6e1374c90380d193044","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-07-09","stderr":0.0},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7RjsANKwwsePSstcZeSMhX","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"af444f0831bcc1693c151ed626c6f9124e9ea5e83544fd5e9442f46ec348b72b","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"e69cdb28e936a021f6d3cd4ba52f95a76e414fed701b1dc1ffbe729c5f0618c0","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:42:21.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:42:21.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"07136419b36b826ece9cfbb80e3e58854c6018372d2a98d092322e3768d30c52","metric_details":{"best_score_across_scorers":"0.9555555555555556","model_release_date":"2026-07-09","stderr":3.1067790907534736},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Mifz96Jm442bn43Qnxj5rT","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMifz96Jm442bn43Qnxj5rT.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Mifz96Jm442bn43Qnxj5rT.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5a08baf766f295fae6810d37d65dbcd66edfc1a0136772649c1e10cf43850ef6","result":{"confidence_high":100.0,"confidence_low":89.46626853767874,"sample_count":45,"score":95.55555555555556,"score_display":"95.6","source_stated_rank":38},"run_fingerprint":"db37723c47a804ff482ab82826709f8b5455e279308dd2abfafc1b4d3f0ef97b","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:54:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:54:12.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ea28a29a0d397b3b42a02de4bec16f3a48e2643786dd4d6c70446b819366fc75","metric_details":{"best_score_across_scorers":"0.6888888888888889","model_release_date":"2026-07-09","stderr":6.979205927323109},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"g7czyLHudHegzP9yRD939X","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fg7czyLHudHegzP9yRD939X.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/g7czyLHudHegzP9yRD939X.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"160912366ba2018b306436502bb72e792da2911f2f8f4b5875843fb13da3be11","result":{"confidence_high":82.56813250644218,"confidence_low":55.20964527133559,"sample_count":45,"score":68.88888888888889,"score_display":"68.9","source_stated_rank":140},"run_fingerprint":"0699e74bc0647cf45f9f0aa085e07d8a6127cf56f79ed05227323f4b64ffa639","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T02:45:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T02:45:09.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3e88f96b7e231e5eef3a5c1c0588e138a7ff84e5a1096a87b325ec9aed7d7601","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.8912280701754386","leaderboard_private_problem_count":285,"model_release_date":"2026-07-09","public_example_count":10,"stderr":1.8475392571140503},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"VBpT2WWYKBZtUxG9Z3svGh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVBpT2WWYKBZtUxG9Z3svGh.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/VBpT2WWYKBZtUxG9Z3svGh.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.6 Sol (max)","thinking":null,"upstream_model_id":"gpt-5.6-sol_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"fa566697fc7062a3f4e2935fd2d9854f3b2afcd922216eebe37ea31e3d64830b","result":{"confidence_high":92.7439839614874,"confidence_low":85.50163007360032,"sample_count":285,"score":89.12280701754386,"score_display":"89.1","source_stated_rank":6},"run_fingerprint":"ca324259067cd359fce56794823999033a5ee75c46b021732421c9d8a35cdb1e","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1ed2106aeea82423442f49da8c12d3f4a1100ac074734824aeb74a965b6c4117","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.44,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-sol"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-sol-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"377ce036d8d437ea9dce0284130749a9714e2195ff5b9f063a11d87a867c96e4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.5,"score_display":"92","source_stated_rank":6},"run_fingerprint":"e4e132a9bae574b504945b1221e0ec29c1437f06ee79ff9516e3e9ec95f1681f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0488ac98e6bc84f818a848509847b60dbae340767f1db8ed9bc93937f5f4b19c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.04,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-sol"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-sol-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2397b4564dfd2ac7b481f64d00b654fc4f6ea9138f4d837711e9fa5a496d55c1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.0,"score_display":"90","source_stated_rank":14},"run_fingerprint":"026fe619fa90d56aa4ee37d5e8eefcde5cdaa3f66bf61ca0482d0e270b675975","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c56f6c928ca5f5102e4b19a8950935084fa46071cda3b440fac88659e5136a1b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.74,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-sol"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-sol-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"34cf48e91e4a1a04ce1882ceac94b3d9eaa41f3901cf1104635d3f13ebf01f81","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.42,"score_display":"85","source_stated_rank":27},"run_fingerprint":"1df87c889f6591b8547be8f454ef8459ed2075dcc985206a61ed77c0ccc38e99","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e94fe9f85c8dcd7d6ec06cd16d783016a6fe3824045eef409668e6b5960e938c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.47,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-sol"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-sol-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7ded57d1483807fb94f311852eb1eaf77622028bf87d3ec2d3a94c4a597bae0a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.08,"score_display":"67","source_stated_rank":62},"run_fingerprint":"d9e53a0bf9ac24cb76ab1a21e68c07673238fe7dd99ec24234ef6cef0e0d73d9","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5a05fd0268f490d440e412f9a7bf48b72843b0d14d4fea6f78184a8fd3cccc21","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.32,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-sol"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-sol-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ffbfba8215239421c3680f9df89b64e018ee9f59699e5888cbd03f47ac7b27ba","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.5,"score_display":"42","source_stated_rank":102},"run_fingerprint":"e987d0c9259a2c8cdc01037a413edb50d99177275d89183f6f7c6a4721b9a85d","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"15b5c85bb36e095b45452aaa895f48da5f4d7b42a32044fcba57e36249597284","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.44","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (Max)","source_effort":null,"source_model_version":"gpt-5.6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"066f59279eb3b4508dfc5236243affc3744c7d63ac011a99c8e389a7abd47823","result":{"confidence_high":94.794673,"confidence_low":89.30786,"sample_count":360,"score":92.5,"score_display":"92.5","source_stated_rank":5},"run_fingerprint":"f16e3164207c4c606ed5c9a514709db040d9142fff5472fb0e389cc1c103e449","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0b7eff27f0b481ce33f63c77011f6f2de11116ab1318037ee9bc5552285594c4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.04","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (xHigh)","source_effort":null,"source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"641f8654bdf273056e760a988141f9e256fb6028188c905cfbf16566dcf51a18","result":{"confidence_high":92.689087,"confidence_low":86.466237,"sample_count":360,"score":90.0,"score_display":"90.0","source_stated_rank":13},"run_fingerprint":"faea3f1fdf41ef542a39e13e9eb32036c63238341a14ae3eaa53b686005a3741","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9d36c0d671ff9a209597a2890f56ea16309ecb29cd4e3bf13f50aa234bde0af","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (XHigh)","source_effort":null,"source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"217f6ce925ce89830b4038df76123feb0008e55cd7f1743938775cbe7b5bd875","result":{"confidence_high":92.689087,"confidence_low":86.466237,"sample_count":360,"score":90.0,"score_display":"90.0","source_stated_rank":13},"run_fingerprint":"a7fd96919038bbb9ce88c37d7f47ae21650974de30b12104305403041a5f6619","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e8a3000e3dcecce6eaca73df584beeeb49886cad5e4e3950f1797466b8f05310","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.74","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (High)","source_effort":null,"source_model_version":"gpt-5.6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"8afb8a4cf2785d5fb05c20a007cf56c3bbbd632c3092436ef6d486ec91a1def4","result":{"confidence_high":88.691509,"confidence_low":81.400531,"sample_count":360,"score":85.42,"score_display":"85.4","source_stated_rank":27},"run_fingerprint":"0e7b0c04b43cb0a55ff2bd170ac05d870b5cb0be078307046020596038e8a0d7","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cea81bef8c7a76c6dac16717f819d1218fa2ec8e2ce5dd41cdb902b6fb3aa2cf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.47","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (Medium)","source_effort":null,"source_model_version":"gpt-5.6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"1a6a08cdfa2f60a88998e57e2d9613babe26b072099e0705c1aecf672156e9bc","result":{"confidence_high":71.731685,"confidence_low":62.067639,"sample_count":360,"score":67.08,"score_display":"67.1","source_stated_rank":66},"run_fingerprint":"fd1787bc7e7d20a6b3777b92360adfa202b177c3882c0f461da7f04886105848","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"094149c8c4c26ac73282a5ccecee161ecb054e6064fe1f5b0b07147098c95d28","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.32","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (Low)","source_effort":null,"source_model_version":"gpt-5.6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"52fc29dec330818c50c7969a1544365b7b1251b792107b2f53249b3bf168ce86","result":{"confidence_high":47.659391,"confidence_low":37.498986,"sample_count":360,"score":42.5,"score_display":"42.5","source_stated_rank":104},"run_fingerprint":"b7c3148a0ff594892199fb94d76bc5ef22635e83cf9562b3199de1ccbb9ce238","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":87,"at":"2026-07-09T20:34:41.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T20:34:41.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5a4baed78b71d905cddb541ef73980abff4be391c88012ba549403c0237e6865","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.8048780487804879","leaderboard_private_problem_count":41,"model_release_date":"2026-07-09","public_example_count":2,"stderr":6.265967111543964},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["pro","max"],"epoch_id":"cdWwJSH3jVfrKyhUHmT54T","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcdWwJSH3jVfrKyhUHmT54T.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/cdWwJSH3jVfrKyhUHmT54T.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":"pro","source_model_display_name":"GPT-5.6 Sol (pro, max)","thinking":null,"upstream_model_id":"gpt-5.6-sol_promax","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"aef2c1d1a352ae61e4d0bf1c566c81c6a875a6e899fa72e7197aef2bf59b2c49","result":{"confidence_high":92.76910041667496,"confidence_low":68.20650933942262,"sample_count":41,"score":80.48780487804879,"score_display":"80.5","source_stated_rank":12},"run_fingerprint":"e6629347fd4fc33bc2d0c41c15b1901aafeb081e89f2e5218e72716b786c4a27","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T02:45:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T02:45:09.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"12bf60a86ca3784a0ec70536dd2f9715c7295c2c8e35ccaf769f2147043a9b1e","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.8292682926829268","leaderboard_private_problem_count":41,"model_release_date":"2026-07-09","public_example_count":2,"stderr":5.949419959829497},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"QJ5rJMVypB4PMxPcBGftc8","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQJ5rJMVypB4PMxPcBGftc8.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/QJ5rJMVypB4PMxPcBGftc8.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.6 Sol (max)","thinking":null,"upstream_model_id":"gpt-5.6-sol_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"320496adc84cee6c4c0e99ae41a3e0a2af74e2403cc0c04e31d950be3a92e0c3","result":{"confidence_high":94.5876923895585,"confidence_low":71.26596614702686,"sample_count":41,"score":82.92682926829268,"score_display":"82.9","source_stated_rank":10},"run_fingerprint":"99997791d332479c55d72dec43951dc99e34c3e60c13cf24b61e373340ef23f5","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"43dcf3a92516241e1ae4b6e3b01dc4e930bd85f291d2505a37d4e07733b4d2f5","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.055795,"date_is_proxy":true,"latency_seconds":26.198,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.757},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-sol"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"11b1f61fb7abaf53b9323d9ec4801421b4be0d13a1df9a0e171ab40347a5d994","result":{"confidence_high":90.32772,"confidence_low":87.36027999999999,"sample_count":null,"score":88.844,"score_display":"88.8","source_stated_rank":6},"run_fingerprint":"039d57de0d418e46ccb39100196500a9fdce7bfe8f9a9ea75e14c06c5e9c61bc","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3df3e799ab386aadfa2ba64a1fc96630e9313fc12149c3355159c9ab0140c4df","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (max) · mmmuPro"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol","upstream_model_label":"GPT-5.6 Sol (max)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0bed35ad738d7dfdbe41eda1efe02d91c9752e39bfb38f009a74a9625c60e6ab","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.410405,"score_display":"83.4","source_stated_rank":null},"run_fingerprint":"c7ee5c2b53737ace1d4726390fa0f910a0b6cb6ea851c9fc00833661a8600626","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (max) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b92e0a48eecc1faed36d21107593521f343b00bc387b3f4b91f3fe882f952227","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · mmmuPro"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-xhigh","upstream_model_label":"GPT-5.6 Sol (xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"67966f81fd4ad32a4e71a322096d68086e786ccdcba44b1669564171a7eca63b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.65896,"score_display":"82.7","source_stated_rank":null},"run_fingerprint":"a7c318ce1f6181fd35edd235d9009a38efe5e1ea81b86f1bfe568130eadf9bfe","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"08f98b84c8812a1a7ae11583c19a0222a4676b3e87dc63316ea046c7c6bee897","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (high) · mmmuPro"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-high","upstream_model_label":"GPT-5.6 Sol (high)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b563b831a2cdb2bdf66b40f7fb5600cdeea76dbc1ba96527b77381d093d8111a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.849711,"score_display":"81.8","source_stated_rank":null},"run_fingerprint":"4848192d26943c4f4e21152e5d684b7df98b405165f3b2ccf6f57b388aada109","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (high) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"76380eb5fba57c6d229d08961f1c4d726e0385965eeae0a0912cd60cdc1ad6df","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (medium) · mmmuPro"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-medium","upstream_model_label":"GPT-5.6 Sol (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"cb81a8decf64924836de100f66c545380fb4ff7b39cf0a89c410aaa12427b3af","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.387283,"score_display":"81.4","source_stated_rank":null},"run_fingerprint":"56e238531a01ef4e8ca99ba9da3c8d96b8ce1c32d2430953af6bbc500af40534","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1311bcaaa737343986e2a93abbfdb06a5a184be74b8bb6fa541601a761e4b457","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (low) · mmmuPro"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-low","upstream_model_label":"GPT-5.6 Sol (low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"55e74e778c2e297f66926bb0c88ce06de20e3b2c2af30fcff18e7c1736b08e10","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.982659,"score_display":"81.0","source_stated_rank":null},"run_fingerprint":"3683deff1ca1bf14511c11ab8b950b46ba10dd08a0d7adced5ce6c24ba2ffdd4","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bf21fd3c94960add0c280fc8ad1e8bb201770f3bcd8448ccf49ed63efa4040ac","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · mmmuPro"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-non-reasoning","upstream_model_label":"GPT-5.6 Sol (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2a66fc6fbea28477caf2b517e7d5a1947a35296da7737ec0379f71e757187cc4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":71.907514,"score_display":"71.9","source_stated_rank":null},"run_fingerprint":"33c12931bc6e0c5026fa97d215f847665386548e30e801993edb0ab1ae7a710e","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_39a5bff9a48a29bea5ce760d","split_or_window":"standard-no-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"e76bb3d6928b6e91fff357a81e942e62719b8c1f482e1821558c3d75f5504cb3","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Multimodal · MMMU Pro"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7668a8bdde63b0a1b008fd1140ee89a7608afe0b024f589d244f127d83c35ea3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.0,"score_display":"83.0","source_stated_rank":null},"run_fingerprint":"4191ec03509c5b719a33cfabaa0853b8aade7a9773051b905bf8483a39eb1e35","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Multimodal · MMMU Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f01085c2078d97f9df207606","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_aa0096763ba73cd85e46674d","split_or_window":"standard-with-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"d9a4f1df95a01eacff2dd545d974234dcc89dc3a4748f81bef27b345696987e6","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Multimodal · MMMU Pro"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"tools enabled"},"run_count":null,"scaffold":null,"tools_allowed":"tools enabled"},"protocol_fingerprint":"c9869f62f1ab1e482460e86f9c0ac73683c41febdcda4cd23712ea1098430062","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.6,"score_display":"84.6","source_stated_rank":null},"run_fingerprint":"c3e3209655bed6af135de1216a5c8d24b201caa4b9e67461031839d18dceca8f","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Multimodal · MMMU Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f01085c2078d97f9df207606","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"eb90a94fca9484bae9a2e65096633ee846be06e0e593bc91970f04a2c121322f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (max)","source_effort":null,"source_model_version":"gpt-5.6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"e210809ec9868cb1447a567e187af0f9a3ba321f024131c231aa9c9c01ff7010","result":{"confidence_high":43.827202,"confidence_low":22.562766,"sample_count":71,"score":32.2857142857143,"score_display":"32.3","source_stated_rank":1},"run_fingerprint":"7ba54530c672c2e0adb209c4b3e442b26f26d9c36d7a361331df7368cacb838a","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e7330f413be4905ebe4d7a5b1556bad37a64b49aeaa19ea28cfc7b6ac012092b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (xhigh)","source_effort":null,"source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8cc086f7e921f89c6bd441cce2fc21a9e78be589901cfa4a89733bc51a48bfc2","result":{"confidence_high":39.995252,"confidence_low":19.401661,"sample_count":71,"score":28.599999999999998,"score_display":"28.6","source_stated_rank":23},"run_fingerprint":"1088cabc5ed8a27b30a6f9a99ed8237fa340b2feaec319c4aa9a22bcff6c4cab","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"97e08d3221d84d7b405ff93d58526e248c8b24b1edfc32dd553a8828b74453f8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (high)","source_effort":null,"source_model_version":"gpt-5.6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"e2c09a0d609be5ad63fffdf5c894a284594375ff61b9cd093aa7b36d67190fc2","result":{"confidence_high":36.941058,"confidence_low":16.980673,"sample_count":71,"score":25.7142857142857,"score_display":"25.7","source_stated_rank":35},"run_fingerprint":"66567bc08145cafae05be8751a806eb387b4055d669172728dab9f8ebcbc77d5","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a370697f6a392ee6346044260623cc0553060db6c79f586ce19ea1249baa45f1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (medium)","source_effort":null,"source_model_version":"gpt-5.6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"68c0b1b52709ef1001caec6f2008b94b64a88a3d5cc0762810fe9f301aaa4679","result":{"confidence_high":33.865422,"confidence_low":14.635335,"sample_count":71,"score":22.8571428571429,"score_display":"22.9","source_stated_rank":46},"run_fingerprint":"603aa6a48d0ab46ceee4bc36b14f1058f9b4b995e363400ecdcb1ed55745de5f","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"73b1156ae8aecee36c8a024bb13c1dd4172513ab4cb660f7e3f26ed66d1567b1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (low)","source_effort":null,"source_model_version":"gpt-5.6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"db1d4e40bd24e27654b16361985adc19f890aa9363076e757fac23aec7a17623","result":{"confidence_high":24.91844,"confidence_low":8.403594,"sample_count":71,"score":14.857142857142899,"score_display":"14.9","source_stated_rank":79},"run_fingerprint":"f35173826cc795fc5491a79707211c8d8824da23f7e55f102e4eea27e76fc442","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fa7d71a9c7b62adebe8ab590a18a853ea659d5b1de86a45c826674d57d86b357","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (Non-reasoning)","source_effort":null,"source_model_version":"gpt-5.6-sol_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6ad074b45ad9982706b31d58f93f1ae9ffefa45361da79a61b0f4d5e4a203430","result":{"confidence_high":12.953726,"confidence_low":1.936999,"sample_count":71,"score":5.14285714285714,"score_display":"5.1","source_stated_rank":110},"run_fingerprint":"1213de4a1af949e4197aa8b78948dbe833762bb47fc46db2e62246ca0bc9c81b","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"970cee7fded3e0607ce670d93d0dd6ca54ec8fcaacf71dc12f84cc29a019e6db","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (max) · critpt"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol","upstream_model_label":"GPT-5.6 Sol (max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"30987dd7ca406bfac5e9eeeb55d1c9bb997ccdb72b2a9c109acf3fc63d20d6fe","result":{"confidence_high":43.911797,"confidence_low":22.502798,"sample_count":70,"score":32.285714,"score_display":"32.3","source_stated_rank":null},"run_fingerprint":"2ae0b65f73573bf04abfe99603c287e99512f9c1cba22a5c2d0c5bc12d31fcf6","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cbc03826dadc80a13b1753d6b996fa90d0320e0f4123334dd94ba156fbb5bbe3","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · critpt"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-xhigh","upstream_model_label":"GPT-5.6 Sol (xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c917901f55ca04b28ecebaa80f0e8cc8032376e87dbca53942c95f06087c6db7","result":{"confidence_high":40.050418,"confidence_low":19.322077,"sample_count":70,"score":28.571429,"score_display":"28.6","source_stated_rank":null},"run_fingerprint":"f5ea42d29ff7779b4b4d0a65272cc28d025443ea3e36eb8e260c57ec35f5d6cc","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"727b89cc5a66ca4d897dc13056df04c27955d351bea2faeb24fa9e38c4a3cdba","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (high) · critpt"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-high","upstream_model_label":"GPT-5.6 Sol (high)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6500601614924cd90f92a5246ac00366143cf53a6a5e5f6840050bc68830414d","result":{"confidence_high":37.026352,"confidence_low":16.929142,"sample_count":70,"score":25.714286,"score_display":"25.7","source_stated_rank":null},"run_fingerprint":"a3692cd8c84cd3e686b3cbcdaef4cdaa81a899886b3a0e78e838bcdb2233df9c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (high) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5f30653f2da08ff3a9d816ef4a07eb9724b1241832558bc5414143b3aeb83355","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (medium) · critpt"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-medium","upstream_model_label":"GPT-5.6 Sol (medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5b7eb60004f5590a622aa89ee19348949baba92def434e5512329dd8f41c7020","result":{"confidence_high":33.950553,"confidence_low":14.58794,"sample_count":70,"score":22.857143,"score_display":"22.9","source_stated_rank":null},"run_fingerprint":"cf884df208314a25dd59dfd9aaca4b961bc985d6bfc23e507a61ab3d56ab00aa","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"04ce2629612b0e63b7832fbd869edce7073ef630b12c3dfd1138a1cc5fe8435e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (low) · critpt"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-low","upstream_model_label":"GPT-5.6 Sol (low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"16ed7390042f52efc6db82629a0745e2b2ed50335de981f88fc0a3b74e11940e","result":{"confidence_high":25.00128,"confidence_low":8.369612,"sample_count":70,"score":14.857143,"score_display":"14.9","source_stated_rank":null},"run_fingerprint":"7e59baa971164f930db200da241ebd3278a25a0b9701763d854f1ae0c79de0c9","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ee34370fe0b2fb1f152d80bc919e2f677b4929500ae27e5f4ebd5a0c622147fe","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · critpt"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-non-reasoning","upstream_model_label":"GPT-5.6 Sol (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"ce13204dcb255a5fab5ce432ead44feeead7eeda58d3b6047286978a5af4783a","result":{"confidence_high":13.028638,"confidence_low":1.924451,"sample_count":70,"score":5.142857,"score_display":"5.1","source_stated_rank":null},"run_fingerprint":"a5d21781082d9f520ea752e5dfef80b712fa10c195935d71b28a03cfab19eeff","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9f878574e2ced8e7daf9eb4daa00f03129305c161b8cf88f852f7935b2b37342","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"f757eca0faeb48281cd4178307e2d0dce127d281f981d358abfdb43cddd15570","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.8,"score_display":"64.8","source_stated_rank":17},"run_fingerprint":"22ec3383a5d6aa3af882092a34f5faad305b28f3e0aa5a423ead6f03a2210dba","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1ca16cff3fb006a2313005176562dfffa73b0405c34e222b89ffa7117a28b633","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-sol_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"48cee48d5cebcf958d61d398772033256fb197c9369b63f9378348a5dc99a73c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.8,"score_display":"64.8","source_stated_rank":17},"run_fingerprint":"023e3978b93d73246d3c716361230a7597dc60975ab7304a1a3e6dcfcef4fe2b","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5cbef7fb02b480c6cc23790a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T02:07:13.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T02:07:13.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"77f2244dd1065b8367b7e43b9face39a569b08937c62384c30d3e2421cb14f0b","metric_details":{"best_score_across_scorers":"0.55","model_release_date":"2026-07-09","stderr":5.0},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CVuujQbCY38ZStXnBcFex9","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"061e90f6357d08257e5d738a737565ea4f9e87110039863937274099b6258baf","result":{"confidence_high":64.80000000000001,"confidence_low":45.2,"sample_count":100,"score":55.00000000000001,"score_display":"55.0","source_stated_rank":7},"run_fingerprint":"cb31530f1c2bff57805d912b31fbbb092fac9056e686af34dfad899c073b4161","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:50:10.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:50:10.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"eb29343a08e94faf35f3af1b6d29081671b81cc85bf3b0be5f63033fa2c285f3","metric_details":{"best_score_across_scorers":"0.27","model_release_date":"2026-07-09","stderr":4.461960433384738},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7cL3BzoAaiHmdMNRCVLE6s","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7cL3BzoAaiHmdMNRCVLE6s.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/7cL3BzoAaiHmdMNRCVLE6s.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"52621b590024685db269251e80944d498f5a4c108eb9ad7eac16fa1bba13d785","result":{"confidence_high":35.74544244943409,"confidence_low":18.254557550565913,"sample_count":100,"score":27.0,"score_display":"27.0","source_stated_rank":59},"run_fingerprint":"d8b6bd0e782a62eb5ec9b6e25acd0b3e132a835891a20cb9d4b709b5869bf32c","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T01:03:13.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T01:03:13.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ecfa1a616eab3484ee022fff68197a19d8aab4f7fe66c2dcf6f2631b216be3e2","metric_details":{"best_score_across_scorers":"0.07","model_release_date":"2026-07-09","stderr":2.5643239997624274},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"KFAFU6kX7qVekqq6QR524K","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKFAFU6kX7qVekqq6QR524K.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/KFAFU6kX7qVekqq6QR524K.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b731a44219b18aa292d0175c80cdd1d3603127bffa4af235cf76d17ac6bfa7af","result":{"confidence_high":12.026075039534359,"confidence_low":1.9739249604656433,"sample_count":100,"score":7.000000000000001,"score_display":"7.0","source_stated_rank":148},"run_fingerprint":"815ce0157292b1e348373a39517420236bafd47f48f77c663e7e3b05e548dd4d","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c65871b1423431f39a5e266ed8cb4681f42b6534ea7ef84f32c61d4145e7a0ad","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.54","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (Max)","source_effort":null,"source_model_version":"gpt-5.6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ec9e359c509b8ca4781d0a7f598536b151c1e22afcc18af93c0e4413ebff89f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":96.5,"score_display":"96.5","source_stated_rank":21},"run_fingerprint":"8ef6ec1e54532ad932a987f103e045e00a9697f76a568b56c7b6a9998ad9dad5","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3bb7b52f788c8798d1dc515d0b0a8459c16a609281cae3bed11a35c1dbe76ce9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.4","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (xHigh)","source_effort":null,"source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5638de844f47721d49bf8fddcec7b9bfbc39efe0c98ec25da0300d0cb7b36862","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"ea226a211dbbc37c195e6bd11bd6e58a2f9a782dda1a7792a7d8333c364b78f6","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4d9378d3a38d9977574342ea341aa49e1a705f983cab1a99bd09043c354d8278","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (XHigh)","source_effort":null,"source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e950f5c2918fdcc6cd8b488480e8d4daa038f16cfb972c30a8dc54bc1ef0f957","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.5,"score_display":"97.5","source_stated_rank":9},"run_fingerprint":"1ae0d80a391e651414695b3291cd889ab23cb5b98ea58759b969a091f75bcd4a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e3b2de421bd57c3fd24e8f3ed6060182222beb116d9c866aae5693d6a3f8f5c1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.3","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (High)","source_effort":null,"source_model_version":"gpt-5.6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"993a6b9ca03c5c1d5529d871d66ddbc3b4d50778ee15132d3b7c86fd9a365b23","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":97.0,"score_display":"97.0","source_stated_rank":20},"run_fingerprint":"24bd87ecf100a6b4548cfe969b78959f730d7f6dad332d1b1591c213072e7608","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c508a97b129e0c3a55ae70af1c3e4e931e4d87bd1bb8da36a2911fc81047543d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.22","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (Medium)","source_effort":null,"source_model_version":"gpt-5.6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6efbdd1a155e9fdb2b4cd6987a2ab9d2a2381ad04ad0a04306ae03e7f9080924","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.5,"score_display":"92.5","source_stated_rank":54},"run_fingerprint":"4c0522695cf878c474b485233ebc6752caf68dca3e149dac624969d7313b75fa","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d1e85079e51a76fd97688b1c945ffdd72ec7342fec5e58197daf328f3669b622","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.17","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (Low)","source_effort":null,"source_model_version":"gpt-5.6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"251f78d340131402ba1ad6852930267feaa7c9d08e68ab3110d82bc16208efaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.5,"score_display":"74.5","source_stated_rank":127},"run_fingerprint":"3deee4def89c5261fcf50a0e84fb7ef15dca6ffa2c528273d7d748cecf884c26","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"638ddf4b1ca001ca34abea2960b26f58365d8ea7bea03d1daba172fd92d216a9","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run; vendor scores using a different agent or tool setup remain separate supplemental runs.","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","cost_per_test_usd":1.1513,"date_is_proxy":false,"latency_seconds":182.365,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/openai_gpt-5.6-sol","source_locator":"Results · Overall accuracy","stderr":0.856},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"38123a473c109a60f405500b38c6a1f4d83afbf7ffcfb1c58d98a37defa21077","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-sol"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"a8c170eb737837c5bd8b2ecfa3634bb4bcbfea4863982e378ada310cf06e5683","result":{"confidence_high":97.87776000000001,"confidence_low":94.52224,"sample_count":500,"score":96.2,"score_display":"96.2","source_stated_rank":3},"run_fingerprint":"385d766acd3a1e57ef16bc2457d7cf2d192cb0de74ab4148d36965ed45316501","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"50975ff78dcb4886883ae6a8fbd9a79df23e9f9a9483f8c461f8837e939da1ad","metric_details":{"model_release_date":"2026-06-26","pass_at_5":79.27927927927928,"potential_contamination":true,"sem":1.833062157601424},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"contamination-risk","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"GPT-5.6 Sol [medium]__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"d16ae687df2d5dd76d6c38924008b1ef9c7b3d1d6fc457424d2d15395e17086a","result":{"confidence_high":65.93514417124113,"confidence_low":58.749540513443556,"sample_count":null,"score":62.34234234234235,"score_display":"62.34","source_stated_rank":5},"run_fingerprint":"041f5582280837b3009bb70c48ba8be0b3667954e59efb13e41b8aa67b072567","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"contamination-risk","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-live","metric":"resolved_rate","name":"SWE-bench-Live · Lite","release_id":"release_9d2de9d5be7e9822af19f7b3","split_or_window":"lite-300-python","unit":"percent","version":"SWE-bench-Live Lite"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":9,"at":"2026-09-26","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:18Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-26","status":"fresh"},"measurement_id":"72952454bae455226fda18bbcea6a72a019147a6d4ef144e520bdb466e78a183","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 300 scored items with a 95 % Wilson interval.","leaderboard_script_url":"https://swe-bench-live.github.io/leaderboard.js","publication_status":"cleared-with-attribution","resolved_instances":211,"submission_url":"https://github.com/SWE-bench-Live/submission/tree/main/submissions/lite/sapient-slingshot-agent/v3.4.0/gpt-5.6-sol"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"Slingshot-v3.4.0","report_file":"reports-0605.jsonl","report_set":"lite","source_model_label":"GPT-5.6-Sol"},"run_count":null,"scaffold":"Slingshot-v3.4.0","tools_allowed":"agent-specific coding tools in the official SWE-bench-Live harness"},"protocol_fingerprint":"bd7ad59d365892c2706fedf0a6bdbe7910b502a8d50f3bffcbda83cdd20c5567","result":{"confidence_high":75.218945,"confidence_low":64.933555,"sample_count":300,"score":70.33333333333333,"score_display":"70.3","source_stated_rank":null},"run_fingerprint":"8862dacc8887efb440f9bc30487e4c319522fae728ad4d3602cefb91ca880b00","source":{"content_hash":"e0226b6333c547885c15c6d53074d0ad5e2a7b99db2d6b54640d7d6537c242b4","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-09-30T18:53:18Z","homepage_url":"https://swe-bench-live.github.io/","id":"swe-bench-live","last_fetched_at":"2026-10-05T12:33:28Z","license":"Benchmark and harness MIT; aggregate leaderboard results redistributable with attribution (maintainers' confirmation 2026-09-02)","locator":null,"name":"SWE-bench-Live · Lite","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-bench-live.github.io/reports-0605.jsonl"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4aea036993309752d798d393","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"swe-bench-live","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-live","metric":"resolved_rate","name":"SWE-bench-Live · Lite","release_id":"release_9d2de9d5be7e9822af19f7b3","split_or_window":"lite-300-python","unit":"percent","version":"SWE-bench-Live Lite"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-31","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:08:10Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-31","status":"fresh"},"measurement_id":"590a282b484fba72639c207d7c5955f358695bb77ee9f187725d03c8630c2790","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 300 scored items with a 95 % Wilson interval.","leaderboard_script_url":"https://swe-bench-live.github.io/leaderboard.js","publication_status":"cleared-with-attribution","resolved_instances":134,"submission_url":"https://github.com/SWE-bench-Live/submission/tree/main/submissions/lite/sapient-slingshot-agent/v3.3.0/20260831-gpt-5.6-sol"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"Slingshot-v3.3.0","report_file":"reports-0605.jsonl","report_set":"lite","source_model_label":"GPT-5.6-Sol"},"run_count":null,"scaffold":"Slingshot-v3.3.0","tools_allowed":"agent-specific coding tools in the official SWE-bench-Live harness"},"protocol_fingerprint":"83155ec5fef29bfbc7984e59fcc34e0a3cb0eaa89922f11fe742fc561c802f3d","result":{"confidence_high":50.32458,"confidence_low":39.143617,"sample_count":300,"score":44.666666666666664,"score_display":"44.7","source_stated_rank":null},"run_fingerprint":"161c3d499787fc953073cf5adeea0c1f88aa05d23ba285e2159a7299942e79ae","source":{"content_hash":"e0226b6333c547885c15c6d53074d0ad5e2a7b99db2d6b54640d7d6537c242b4","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-09-30T18:53:18Z","homepage_url":"https://swe-bench-live.github.io/","id":"swe-bench-live","last_fetched_at":"2026-10-05T12:33:28Z","license":"Benchmark and harness MIT; aggregate leaderboard results redistributable with attribution (maintainers' confirmation 2026-09-02)","locator":null,"name":"SWE-bench-Live · Lite","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-bench-live.github.io/reports-0605.jsonl"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4aea036993309752d798d393","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"swe-bench-live","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"2ddc513246078876bbb8f0dd3ecbc4fb0805cc1d5a78b1773874f253b5429e37","metric_details":{"ci_attempted":450,"ci_half_points":2.829822249837175,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":327,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":61.25333333333333,"mean_cache_tokens":7419435.235555556,"mean_cost_usd":8.386436346666667,"mean_duration_seconds":1128.624910331111,"mean_input_tokens":7907652.344444444,"mean_output_tokens":60013.64444444444,"median_agent_steps":53.0,"median_cost_usd":6.8414715,"median_duration_seconds":1013.6291865000001,"median_input_tokens":5973865.5,"median_output_tokens":58786.5,"median_output_tokens_to_pass":58467.0,"median_peak_context_tokens":177470.5,"n_attempted":450,"n_passed":327,"n_tasks_attempted":113,"n_tasks_passed_any":97,"pass_at_4_points":85.84070796460178,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_sol_max","source_model_version":"gpt-5-6-sol","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"a712907929a4595a3adce2f6ebe5586fc1817c713049c58fbf69d2094cee7ffa","result":{"confidence_high":75.49648891650385,"confidence_low":69.83684441682949,"sample_count":450,"score":72.66666666666667,"score_display":"72.7","source_stated_rank":9},"run_fingerprint":"367118f965c334b7db4546c11f83474c41c3f1666cff04c2e78b63a752e82c30","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"6cec4507d1025402ed3e80e6b52ded7da30a8008e1bca18fc88bb0f8589ad0f3","metric_details":{"ci_attempted":451,"ci_half_points":0.8191117217343103,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":319,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":44.0,"mean_cache_tokens":3967289.3303769403,"mean_cost_usd":4.703663860310422,"mean_duration_seconds":800.5459875388026,"mean_input_tokens":4259318.379157428,"mean_output_tokens":40744.59201773836,"median_agent_steps":39.0,"median_cost_usd":4.117039,"median_duration_seconds":694.028368,"median_input_tokens":3343837.0,"median_output_tokens":39449.0,"median_output_tokens_to_pass":39434.0,"median_peak_context_tokens":133054.0,"n_attempted":451,"n_passed":319,"n_tasks_attempted":113,"n_tasks_passed_any":97,"pass_at_4_points":85.84070796460178,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_sol_xhigh","source_model_version":"gpt-5-6-sol","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"fba9df76b7bdece3da4901cf816ff3f525398685424448174e94d3d7beb2e91d","result":{"confidence_high":71.55081903880748,"confidence_low":69.91259559533886,"sample_count":451,"score":70.73170731707317,"score_display":"70.7","source_stated_rank":11},"run_fingerprint":"f508842450016cb82ed45040eb877bf5d1e95841e58a60aa512667b236991e37","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"bd380050d0a7f824a92db4614ae633af500d47c55c164fbfedafa013b88d9203","metric_details":{"ci_attempted":451,"ci_half_points":1.4321452203772356,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":313,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":36.889135254988915,"mean_cache_tokens":2435251.370288248,"mean_cost_usd":3.4698331108647453,"mean_duration_seconds":594.230510789357,"mean_input_tokens":2713572.2505543237,"mean_output_tokens":28450.31929046563,"median_agent_steps":32.0,"median_cost_usd":2.979421,"median_duration_seconds":517.298681,"median_input_tokens":1979416.0,"median_output_tokens":27700.0,"median_output_tokens_to_pass":27706.0,"median_peak_context_tokens":98355.0,"n_attempted":451,"n_passed":313,"n_tasks_attempted":113,"n_tasks_passed_any":98,"pass_at_4_points":86.72566371681415,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_sol_high","source_model_version":"gpt-5-6-sol","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"0205a1a53c4a6bbc70fb38ae30a8710ea18fd20d88f0ce98d6323b438bd838a1","result":{"confidence_high":70.83347559731736,"confidence_low":67.9691851565629,"sample_count":451,"score":69.40133037694012,"score_display":"69.4","source_stated_rank":15},"run_fingerprint":"37f6869be49f217cd9ec04244fb1d7fe914340441bdd4f36a879d535a90eefd9","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"27d79ba99eb5b2d2b129f8e93b7eb5f178ac25646d0cedceb38a31117f69b51a","metric_details":{"ci_attempted":452,"ci_half_points":1.583357649307215,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":276,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":30.913716814159294,"mean_cache_tokens":1382962.9734513275,"mean_cost_usd":1.8620318075221238,"mean_duration_seconds":423.3752944181416,"mean_input_tokens":1505793.9557522123,"mean_output_tokens":18425.216814159292,"median_agent_steps":26.0,"median_cost_usd":1.545414,"median_duration_seconds":354.895511,"median_input_tokens":1064552.5,"median_output_tokens":17645.0,"median_output_tokens_to_pass":17452.0,"median_peak_context_tokens":65185.0,"n_attempted":452,"n_passed":276,"n_tasks_attempted":113,"n_tasks_passed_any":91,"pass_at_4_points":80.53097345132744,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_sol_medium","source_model_version":"gpt-5-6-sol","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"fd2916412cfc99b2adf128c8d1c16558113436dd0634446e5918b53d8922d05b","result":{"confidence_high":62.64530455196208,"confidence_low":59.47858925334765,"sample_count":452,"score":61.06194690265486,"score_display":"61.1","source_stated_rank":32},"run_fingerprint":"d40f7b8d1d7ce369f530c06da462eb020cff7fefe756739cb8496f542a03ebb4","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"8c8b839e5f39432def62abcbabe30a0c12bd3c6dc234fe066cd3b7711ec75798","metric_details":{"ci_attempted":452,"ci_half_points":2.38819467145892,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":205,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":23.358407079646017,"mean_cache_tokens":599908.814159292,"mean_cost_usd":1.0742664646017699,"mean_duration_seconds":262.2510164579646,"mean_input_tokens":690833.657079646,"mean_output_tokens":10579.141592920354,"median_agent_steps":21.0,"median_cost_usd":0.944557,"median_duration_seconds":211.1408625,"median_input_tokens":541018.0,"median_output_tokens":10062.5,"median_output_tokens_to_pass":10031.0,"median_peak_context_tokens":39823.0,"n_attempted":452,"n_passed":205,"n_tasks_attempted":113,"n_tasks_passed_any":81,"pass_at_4_points":71.68141592920354,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_sol_low","source_model_version":"gpt-5-6-sol","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"91dc4a6bc5bb6a432e172605ab46266dc9e71ab242c5063547cb18b94690a6ff","result":{"confidence_high":47.74217697234388,"confidence_low":42.96578762942603,"sample_count":452,"score":45.35398230088495,"score_display":"45.4","source_stated_rank":54},"run_fingerprint":"faadcca50451e6de9d8d167cef448debc588dc7a18d7d14b6a6a1231e44f7057","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f730568fd454a1ddf62a7130da1314e0d83bd9b38926315f7c28d82a47e3c711","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"61.25333333333333","mean_cost_usd":"8.386436346666667","mean_output_tokens":"60013.64444444444","model_release_date":"2026-07-09","notes":null,"pass_4":"0.8584070796460178","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-sol (max)","source_effort":"max","source_model_version":"gpt-5.6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"edf5c7090f83da95e1cd5529ec2b8a3733fa7a67a286493950c7f3ff4fa71eaf","result":{"confidence_high":75.49648891650385,"confidence_low":69.83684441682949,"sample_count":113,"score":72.66666666666667,"score_display":"72.7","source_stated_rank":9},"run_fingerprint":"db3aba0f92648aef807ea271ee5d4448f090a2110aed47806e08a47be9abdd11","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"be177b55a68822cd0bdba9aa81966ae83bbcc74c861e9f0af1ef87aa06ac92b2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"44.0","mean_cost_usd":"4.703663860310422","mean_output_tokens":"40744.59201773836","model_release_date":"2026-07-09","notes":null,"pass_4":"0.8584070796460178","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-sol (xhigh)","source_effort":"xhigh","source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"3dbfbca30d18491079b2e56a8f1b3c85ca5170ddc897788d5f59bc743376374c","result":{"confidence_high":71.55081903880749,"confidence_low":69.91259559533886,"sample_count":113,"score":70.73170731707317,"score_display":"70.7","source_stated_rank":11},"run_fingerprint":"c1ea6701577128df1deb3f827d08deda44c2ba09391bf724d49482995a36b005","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bfebd7f231e41fcf5ce37ac1c6731af4228514bdf54ec062dbf45b058814178f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"36.889135254988915","mean_cost_usd":"3.4698331108647453","mean_output_tokens":"28450.31929046563","model_release_date":"2026-07-09","notes":null,"pass_4":"0.8672566371681416","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-sol (high)","source_effort":"high","source_model_version":"gpt-5.6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"eae2e39686e97cb0f433f463768d7f8930fa7e8dddb2e6a30d9fb17cad42d29e","result":{"confidence_high":70.83347559731736,"confidence_low":67.96918515656289,"sample_count":113,"score":69.40133037694012,"score_display":"69.4","source_stated_rank":15},"run_fingerprint":"b79907bb99835d9b254ec949b7427289572b77fe1237c20b7ffd718135a651c7","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7c4ca7712f7e7cce235cd85fb7c0680d75057d88eb804a0f87fe966cf4b13c9d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"30.913716814159294","mean_cost_usd":"1.8620318075221238","mean_output_tokens":"18425.216814159292","model_release_date":"2026-07-09","notes":null,"pass_4":"0.8053097345132745","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-sol (medium)","source_effort":"medium","source_model_version":"gpt-5.6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"b08e50af2da1e513c4f31afc038c1a071f1cffcae9084bf4b8b944692ffaf426","result":{"confidence_high":62.64530455196208,"confidence_low":59.47858925334764,"sample_count":113,"score":61.06194690265486,"score_display":"61.1","source_stated_rank":31},"run_fingerprint":"5267e42ada6806e94731ff12b88778d977a4434d2337f56a4b479de7ea8ddc58","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c6942c1060ca5e653a5ec4af7cd974cffdac1d97047a11c4457963e75b35765c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"23.358407079646017","mean_cost_usd":"1.0742664646017699","mean_output_tokens":"10579.141592920354","model_release_date":"2026-07-09","notes":null,"pass_4":"0.7168141592920354","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-sol (low)","source_effort":"low","source_model_version":"gpt-5.6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"760a5228723bf3aa56e65b778fd92559c4f6c88ba006998d02089ce1fa99e0fe","result":{"confidence_high":47.74217697234387,"confidence_low":42.96578762942603,"sample_count":113,"score":45.35398230088495,"score_display":"45.4","source_stated_rank":52},"run_fingerprint":"0eee4c04e5b01bd3a3a6e37f20056c64c5fc214910da1de3e2235380323e3940","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_e361abe5266a3ddfdcef017b","split_or_window":"openai-provider-comparison","unit":"percent","version":"DeepSWE v1.1 · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"f5ca1e996e6c80a78de6e98a59de66cb154e014887a35781241b74e52908a9d0","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · DeepSWE v1.1"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value for DeepSWE v1.1; competitor rows are secondary re-reports; exact scaffold, task coverage and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"coding-agent tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"28bd89a57c6784aa44dda4f48bb1ba1e048300dc4313759e24d750d50b4e533a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.7,"score_display":"72.7","source_stated_rank":null},"run_fingerprint":"90807e634391c0365555da3fab459c8dc4fd3560cd710abd370c862ada5dbd6f","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · DeepSWE v1.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f01085c2078d97f9df207606","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_7664765571d731e0f6ff7adb","split_or_window":"openai-provider-comparison","unit":"percent","version":"SWE-bench Pro · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"cda346e31d33bf692052a2ee845610857201404cf12afa7273e63d7f64310456","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · SWE-Bench Pro"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; OpenAI rows are provider results, while competitor rows are explicitly retained as secondary re-reports; exact scaffold, task revision and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"bb784ac6361cbbe179556c67a1258b8773e5fdda82afa9a06e9f85b5d8df29c2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.6,"score_display":"64.6","source_stated_rank":null},"run_fingerprint":"f847c6e2e9eb53b499e8389951ca1989ef1179f089f5cb5183f08e15536cfbaa","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · SWE-Bench Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f01085c2078d97f9df207606","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"32c2ff46b8b191ad2dce26cf00e56586c00c4cba57edb91a88911d28876fadd4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"8.23","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"99","tokens_per_task":"42944","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol Max","source_effort":"Max","source_model_version":"gpt-5.6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"086937a44a78e6acf30897e6544afd364dbeda70f4da99bdc73c94ba51efad46","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.7,"score_display":"41.7","source_stated_rank":20},"run_fingerprint":"51a47e697bbd2814beb79710072db02ad400eb6269472a7e3cd3e0d7e1bf5013","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"13d28bd31f188ba118eb468de552e537ea2c40c71f40a9e0a753712d0283647b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.4","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"55","tokens_per_task":"24729","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol Extra High","source_effort":"Extra High","source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"c786b64fb9d7ae0373365ce1b20e0cc95a1c6d0e8ac479a55d34f73f01ee71e3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.7,"score_display":"37.7","source_stated_rank":29},"run_fingerprint":"dc366a35ea906257a1bdc8ab83dc6b53baa42653a3a1eadd31c0b105225e7c9e","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bcefcd6f1d9c80fa02ef685199cfd3b118658db372bfaeb897593e4e68b03b39","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.85","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"41","tokens_per_task":"16174","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol High","source_effort":"High","source_model_version":"gpt-5.6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"1556689cd0e4539ec3a7e3a4fdf3f3c286af870a1ff4895aa07904db71f1e47a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.7,"score_display":"35.7","source_stated_rank":35},"run_fingerprint":"2961b5bb0e42b3a7fde84a012614229f597fb454f8e3503e1667575dc09c336a","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b178d9a76ba3527641f27148501cd252377353ed64284de1bf86cf07379d7d5d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.77","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"32","tokens_per_task":"10111","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol Medium","source_effort":"Medium","source_model_version":"gpt-5.6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"bc5e3f1f4fc29c62b26bf7a3b918ffdc76b4ac2bd66264cb42affdabf8621b90","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.1,"score_display":"31.1","source_stated_rank":44},"run_fingerprint":"ab079f829fcb66c83b0deaf1e048ac660e726d7d5724cf03881cd221fc0fd805","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b88bb736915c140df0b7b600da8b0a34eaee22cd8c9715bbed9eabf486f4297e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.87","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"21","tokens_per_task":"4885","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol Low","source_effort":"Low","source_model_version":"gpt-5.6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"c28b69a646f5f34ce787d7c08e5ce2112604ead3b7e7c99e4e3f0b43aada339b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.6,"score_display":"24.6","source_stated_rank":53},"run_fingerprint":"4f44a1160d917db3a4dd843ba214634208f41b648e92b70e56de40231f542b3e","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bae4e6ea809a17188a990c8bb5d2a2e066f5111ecfe6ce4cb3493d4490208799","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol","source_effort":"max","source_model_version":"gpt-5.6-sol_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"codex","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"0b4a3eaa2469018bd79fde26e6816f088ff1492ea6488ebd93af3954346ad7f7","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":47.49,"score_display":"47.5","source_stated_rank":11},"run_fingerprint":"d51b599c1e8b2bc5726e5ee070fb3e192a8306bfa0df4b2046590d98e18fec96","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"433043b7684b275a8005214450de165f40ba8279c7b40de057509e9c1049dd6f","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":7.982135,"date_is_proxy":true,"latency_seconds":3711.631,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.51},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-sol"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"5a03e88de84e3e4fb5b475aefacd8434af18626290cba2c2986691ceb832cb99","result":{"confidence_high":100.0,"confidence_low":82.3274,"sample_count":null,"score":91.167,"score_display":"91.2","source_stated_rank":5},"run_fingerprint":"64314c2d04ffdd4f61e65d28c7076123a87a881d6457a5e1d673f473f10b7900","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"88be8de8d7874a44a8f3b231bcf162b531e09f5b13b1477675f5660ff3abfe89","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.089831,"date_is_proxy":true,"latency_seconds":56.513,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.088},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"max","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-sol"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"d0583d56bf3b58ebe8d422f96d5df8064f1d6f8a54af8a6c344adc58a8a3310a","result":{"confidence_high":84.73648,"confidence_low":80.47152,"sample_count":null,"score":82.604,"score_display":"82.6","source_stated_rank":52},"run_fingerprint":"6d959f6def809807730f3cf79dc729bc4ed3ce05e2a19b69f1991119c5e793c2","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2e489b992fcb5f5990e9dea2cf24be34f1b35e8bce4178c6b47864922fa2f966","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (max)","source_effort":null,"source_model_version":"gpt-5.6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fe69c5c1635e5543edd68b44bacb984a4d461ae6e3630b45a8ebf81001e26365","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.0601851851852,"score_display":"57.1","source_stated_rank":23},"run_fingerprint":"9c11f375c166ffc3fb993ba5d6972fac41946b1533ba76fc8c2aac4a725411f7","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3b2be8112f908c090551a04577ae8093b964d310bcf4361e3257d1ddba03c052","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (xhigh)","source_effort":null,"source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"f759f07fe08accceb2172f8f455af323ade338e5cd1062e8c2629cbe3cc2323e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.0185185185185,"score_display":"56.0","source_stated_rank":34},"run_fingerprint":"26a2f00d0d2c0d3c3144596f325ca9da4d04c1e1994d142bd94f5e5de347a221","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"af4278bbb446f318341b7f3756c516f8f608b765921efd03eda96a644bdf3a1c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (high)","source_effort":null,"source_model_version":"gpt-5.6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"65908b2c3526c59224e44cd153e0a83b30b0f51f73adcd248fbc1fb480dcac50","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.9444444444444,"score_display":"56.9","source_stated_rank":24},"run_fingerprint":"6734d54a3ffd20c5a603de2e9d44745e0f587b7daf7a12895abc4c8ee003347f","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"79678aaeae5e8f97f5ab921500d17adea9d8700fbc31e27b7953be7c6300d5ae","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (medium)","source_effort":null,"source_model_version":"gpt-5.6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"4c18edbada0aca2d69100e70944f4d136718ee197551ea0ff904d0f1cdde0dea","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.4814814814815,"score_display":"56.5","source_stated_rank":28},"run_fingerprint":"b364c4dc5a606eb4927ad0a5b8ff1edef95843c8d2f1d1e461188eda21e800e6","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"10fa44812761643623bf63a22264b20de9fbd7cb743af7cd9a7bbdcae55fba9b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (low)","source_effort":null,"source_model_version":"gpt-5.6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"7b5ed2f96513aa8ea183b6125ba54c9f95babf3b1b849eb1a13720f22d85d5c6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.4398148148148,"score_display":"55.4","source_stated_rank":40},"run_fingerprint":"f6419798c5e612db0157774cc3f8e8b92303dc762a28900323305671f44c1e2e","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f15e128006fd230311985b1eecd4789e5bf71db8f56701cd421cc983178a05e8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (Non-reasoning)","source_effort":null,"source_model_version":"gpt-5.6-sol_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e5342b20fcabb86c2456f316188d82793d2022124e266859a509926ccc0a2e32","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.1064814814815,"score_display":"47.1","source_stated_rank":110},"run_fingerprint":"ee33625dd99f601fd5d28853221e040f8c233a229cbd91ac5b132cd0f425d1c4","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8f8511bdcbcb154223dd306c18ab8583715cfa38832a5785dddea7ca653fd76c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (max) · scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol","upstream_model_label":"GPT-5.6 Sol (max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"5ae75e0e353aa53477a0d510d5dd4316a52974d1b1e3506fc72942cea8d35b67","result":{"confidence_high":62.647105,"confidence_low":51.287394,"sample_count":288,"score":57.060185,"score_display":"57.1","source_stated_rank":null},"run_fingerprint":"66bc52fbde7ce10ad7122ea2b2fc28cf5d4f6e6347e5de8bf9060f398d0aaaed","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d2e4b2f4af96c4bd23cd71283262e1b05b44c71d00687122536315e04281d7ca","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-xhigh","upstream_model_label":"GPT-5.6 Sol (xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"56b62c74b211bfecdec763b6d0b1746ca3175e398b612200ddfab2f26cd64f89","result":{"confidence_high":62.647105,"confidence_low":51.287394,"sample_count":288,"score":57.060185,"score_display":"57.1","source_stated_rank":null},"run_fingerprint":"0b987ecfa663dd514dc124d8ce855320f8f1224022efea1b056093b8b9a84381","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9d44d4d18d6eff48438adce1449c2a96f7d53c3daa7d87838db3b4631fac8618","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (high) · scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-high","upstream_model_label":"GPT-5.6 Sol (high)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ea46b85a70100e6b94f95c9b5e491ea351f740e2fa0dfc71978e157013d6df22","result":{"confidence_high":63.320632,"confidence_low":51.984475,"sample_count":288,"score":57.75463,"score_display":"57.8","source_stated_rank":null},"run_fingerprint":"78bd3b8c6a6cde17542502aaf42382a27adda5b0a48b2e7ad17f0b042101de9b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (high) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f3c5dfeeddf7a05668b71aebe65a3589de028f088f2691dd2665f6936080bb1e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (medium) · scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-medium","upstream_model_label":"GPT-5.6 Sol (medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"a5f84ffef55fd3bf749fd515d40d998cc065b0f89870deb956c9e05eebf4be97","result":{"confidence_high":62.984009,"confidence_low":51.635793,"sample_count":288,"score":57.407407,"score_display":"57.4","source_stated_rank":null},"run_fingerprint":"b60fa8ad33688b170a044885516ef61ee515bb64bfa0d07697c4d7614b6ccf8f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0388a23a8469a4861e76deb900fa55fc049024d427e01143ff925b265eb09b5c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (low) · scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-low","upstream_model_label":"GPT-5.6 Sol (low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"630b171115c5aa394a97ced39b03a3d8549eb5b75e3437a2d59de26df8a6f69d","result":{"confidence_high":61.972454,"confidence_low":50.59144,"sample_count":288,"score":56.365741,"score_display":"56.4","source_stated_rank":null},"run_fingerprint":"e145ac0e5a317f70f8f7fd406286abcdd92094a063244e8cb6f784eaae83c0fc","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":19,"at":"2026-09-15T17:15:32Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-15T17:15:32Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"226c5aa2477844f8b61ade3ddb270b66ddfff7409a6c0fc405a89d2c3e2171dc","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · scicode"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-non-reasoning","upstream_model_label":"GPT-5.6 Sol (Non-reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1754ff2d2d734ef4956fd3e0718134fa3dfc544d21e06f7e52111a36bd040353","result":{"confidence_high":53.446158,"confidence_low":41.985153,"sample_count":288,"score":47.685185,"score_display":"47.7","source_stated_rank":null},"run_fingerprint":"b487edad67622e95be415b9556dd61a9d75f9b0564dca61d17f7417c3c773e91","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"485f4148e4bc60715689c87681098d77516a037e11e0e40e4df6814f75cd0ca3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"1.534078","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.6-sol-max","source_effort":null,"source_model_version":"gpt-5.6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"66845a98dccfd2890bdd6dca1ad15dcc6200b9ed34f3c49cc1f835106236352f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2176.88,"score_display":"2176.88","source_stated_rank":3},"run_fingerprint":"a9f0b495313f8598f5a8801710b3705ef708f3af056efd7e5317451e9a8f4063","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"19e404b6a53778f7f0df5b030ea2eeb4e8b5680f54afd9a60991ee346b28722b","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":33.403634,"date_is_proxy":true,"latency_seconds":2033.047,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.716},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-sol"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"c4fe4577679a1a2dfe2f574d822098705152a702e19e2d71efdfb92ab424bb0a","result":{"confidence_high":87.77836,"confidence_low":73.21164,"sample_count":null,"score":80.495,"score_display":"80.5","source_stated_rank":21},"run_fingerprint":"fc1169fe29af220ede2b129fa2cc808345ec1f52a9d8b2d6548bae290dad9dd1","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4f8e0cea41da0b578616514b89ed225a5a0bc19f9451ac24c7a449dffd1cb10e","metric_details":{"license":"Proprietary","variance":9.373037659021854},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.6-sol-xhigh (codex-harness)","subset":"webdev"},"run_count":null,"scaffold":"Codex harness","tools_allowed":null},"protocol_fingerprint":"e6bbceab137527b614e0ba2f049446fece1c0600a2238bcd90710f2a4c68f6ca","result":{"confidence_high":1625.7675157337083,"confidence_low":1613.7664927453213,"sample_count":17128,"score":1619.7670042395146,"score_display":"1620","source_stated_rank":22},"run_fingerprint":"e51e71b12c1e351036ae1eed9aaed4b5353400763799a4ef66b8d6dcce7ac24b","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"15e2da212223ac4ec25d70479c179da3592bdc3b8f56591feb102de0c0fd5331","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"58022ef2612941aa24df281b77e355c2d4269cb5c5044f3fae6f2ad61fa699b4","result":{"confidence_high":1624.28,"confidence_low":1610.36,"sample_count":11916,"score":1617.32,"score_display":"1617.32","source_stated_rank":17},"run_fingerprint":"fca43984820600efec90b05fdbccf7955f97b9729ede01731debcbab7cf84c98","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"mirrorcode","metric":"mean_score","name":"MirrorCode","release_id":"release_07bf24088c0805f7b6df7a88","split_or_window":"15-programs-30-language-tasks","unit":"percent","version":"ML +Private 2L"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T16:12:44.772Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T16:12:44.772Z","first_observed_at":"2026-08-28T19:57:34Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"462d5b5cca9c6063b9d6cf373508cbb5907c258b729fdfb8339adad3a65b4f0e","metric_details":{"best_score_across_scorers":"0.2","model_release_date":"2026-07-09","stderr":9.229582069908972},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mirrorcode-ml-2l-gpt-5-6-sol","log_viewer":"","logs":""},"run_count":3,"scaffold":"Epoch AI Inspect ReAct agent","tools_allowed":"shell, text editor and visible-test evaluator; no internet"},"protocol_fingerprint":"a58de79cf4f660b386289bc562f0a44ea06f3aa39e01544289220a4104beaad7","result":{"confidence_high":38.08998085702159,"confidence_low":1.9100191429784168,"sample_count":30,"score":20.0,"score_display":"20.0","source_stated_rank":6},"run_fingerprint":"bb02710d8f4eaa93141429a95667179ff9573c2627546757a09a04923c202cf6","source":{"content_hash":"3e212e97e03d0343e676ecc7ff046ee7b94d219ba7d863364634b7361dc89b4b","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://epoch.ai/benchmarks/mirrorcode","id":"epoch-mirrorcode","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · MirrorCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/mirrorcode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0fde9bf73bcd3ca6de62b6e415c6c535247154d0ef22b8501080c155f52d7562","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"service_tier":"pro","source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-sol_promax","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"d10ad421cd3940208257e8f5ba0b686f73dda194844159ca82159e453a2ce7cd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":89.43,"score_display":"89.4","source_stated_rank":9},"run_fingerprint":"64891c201e06d266b46f1b3af88375989591cc321ce6cf989082b6c68e00135d","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"063f7791b6ff743c468dc5d44be0548c9f80670c9a97a7f3da0eb1437d097a78","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"23ebf117c96b90c27f01fb28c9ca382b966525dfcbcc2fa282d288d03934bb19","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.97,"score_display":"87.0","source_stated_rank":12},"run_fingerprint":"2938f36629f1887f040786df04f71e34710cc79bc3a616bcc5fd2d94040a7cb1","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a27c991da31f0cf5aff2a90d2606d1da775560034bcf2532904bffbf9affad73","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"b1a5ceb9f842670f91ef294b77110d916953a36a8e5d23e4ec35d97182266641","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.76,"score_display":"88.8","source_stated_rank":10},"run_fingerprint":"d82b59715e28c309aeb101b5882b266b0ac5db57e7befbcd87503e12fee6b847","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"0478ca66d40c4c3557e3e10b4710b9c25905b7bae6a447a439649eb2bd6f6706","metric_details":{"benchmark_page_updated":"2026-09-28","cost_per_test_usd":1.023319,"date_is_proxy":false,"latency_seconds":372.819,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/openai_gpt-5.6-sol","source_locator":"Results · Overall accuracy","stderr":1.35},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"f10dc0e40e1824a7bdbed4a7fb489580402cd488558667ac010fecb8a3e3746f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-sol"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"72ec2d00d432bc3875dba5eb782e1bd255ad9a8c258affe0088f99049f7a8ec4","result":{"confidence_high":88.414,"confidence_low":83.122,"sample_count":null,"score":85.768,"score_display":"85.8","source_stated_rank":3},"run_fingerprint":"a776d7338d202241767ba043dfda3a304dc72f0253770eb1199b2297be834870","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cf58be460126d3ae667c0899701b86c3fd8138fc9dffe2ee57bccfed7a6c25a7","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (max) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-sol","upstream_model_label":"GPT-5.6 Sol (max)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"a2545a13ac4904e193b9c90ac812010c5f550d3109bb368ae81912d419cd4799","result":{"confidence_high":93.233339,"confidence_low":79.650655,"sample_count":89,"score":88.014981,"score_display":"88.0","source_stated_rank":null},"run_fingerprint":"db2f6a34932f2ea95c7f6dfb65761b7e29c3a9556686b561d332cc067d5795e6","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (max) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5f7f4f95fd4746446c80816acdf795108ed1e541bc4b5ee17fe5ac8157b53203","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-sol-xhigh","upstream_model_label":"GPT-5.6 Sol (xhigh)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"e0318b08537b0f5e17bdbc0a737dda7bf3f3ba09bfeb9dbc958366b7fb4c8825","result":{"confidence_high":94.321369,"confidence_low":81.434902,"sample_count":89,"score":89.513109,"score_display":"89.5","source_stated_rank":null},"run_fingerprint":"4669d863780358bcd15830926fd0e2a0c34c799788f83fda01553085b07ea780","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"69cf75e3e9f036adccf9f017c5024ec00f2b4f972ffed35fb86999aa7c66def6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (high) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-sol-high","upstream_model_label":"GPT-5.6 Sol (high)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"478eda866ce78427d9508ac517474e9acb41d111557d1d18a6aef2c632399048","result":{"confidence_high":92.677999,"confidence_low":78.769859,"sample_count":89,"score":87.265918,"score_display":"87.3","source_stated_rank":null},"run_fingerprint":"2b91de06fde046023617adac89b215a6fbea55f75b972ae62caa60d10371810e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (high) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b39af81bec4d2c48b61526dff959a1ae1f88aa715a9a2b889d5180ce7e911401","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (medium) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-sol-medium","upstream_model_label":"GPT-5.6 Sol (medium)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"10214232d5351e9ac509cbccc48a95885e9b04baac2c0a73197e43945a21cf15","result":{"confidence_high":91.832281,"confidence_low":77.461369,"sample_count":89,"score":86.142322,"score_display":"86.1","source_stated_rank":null},"run_fingerprint":"0cc20344030d8c82cec3ba61722834c7a4bd084a633f3ef93732946c1fc16d52","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (medium) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2ce011b5163a6e96f202c3904ad0c2a2e6f6f8177659a59154330f653bfd8289","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (low) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-sol-low","upstream_model_label":"GPT-5.6 Sol (low)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"af71c8ed3277d51b43422dfbba8202869e7126ab092adb815be4604e05f469c4","result":{"confidence_high":84.331216,"confidence_low":67.010711,"sample_count":89,"score":76.779026,"score_display":"76.8","source_stated_rank":null},"run_fingerprint":"63b1a33bb308d5e7f4722b66614d0fba18b69e3318684a6f7d0022851def5f52","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (low) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3deb2af00aa9137fc058f12f0eb09f204ddc05359ae52928d40f16456b3d066f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-sol-non-reasoning","upstream_model_label":"GPT-5.6 Sol (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"d8224e98cac025e7d124d3b84e1eecd7c2e8350ec000e3fe0168918aca6798dc","result":{"confidence_high":82.118581,"confidence_low":64.196863,"sample_count":89,"score":74.157303,"score_display":"74.2","source_stated_rank":null},"run_fingerprint":"037079a15ee22679fea31891696e41c599ac40c8fb333e1d452e6b2a07f7702f","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_f12b7c28fa713a4b77a0a836","split_or_window":"four-agent-orchestration","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"90cca00f506b3538067b7db54bc166dcc0b9dd92cee7ef1f7a4a156cf5a6c299","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · Terminal-Bench 2.1"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"Ultra orchestration; four parallel coordinated agents","reasoning_effort":"ultra","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"terminal agent toolset"},"run_count":null,"scaffold":"four coordinated agents","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"8a87a53136874c8f9f032e5eb9c2a3afb4b436ffaf22d02ed3b1e0d37fe0af07","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.9,"score_display":"91.9","source_stated_rank":null},"run_fingerprint":"0fc7e7e8960994ec5980f8f5460037de6cc746007e1c8a2fc67022cdb7d508bf","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · Terminal-Bench 2.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e13d5b09e64a59050fc921a1","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"openai-gpt-5-6-report","source_label":"ultra"},"provider_mode_key":"ultra","raw_label":"ultra","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_d522137a2b9b55b4156d183d","split_or_window":"single-agent-baseline","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"94f0300563c4027f8dc61d63a0f0ad736ea963ee935e8d725755ec304fbddc73","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · Terminal-Bench 2.1"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"single-agent baseline; exact harness not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"terminal agent toolset"},"run_count":null,"scaffold":"single-agent harness not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"35ec4343c781d4675f2f64710c5000fe0977214013c6f6f70121275e1c4fe180","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.8,"score_display":"88.8","source_stated_rank":null},"run_fingerprint":"82753a67febbeab9d13abfeb83f690508aa624321f8d2a610baf591e7f52b143","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · Terminal-Bench 2.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f01085c2078d97f9df207606","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T21:39:34.661007+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T21:39:34.661007+00:00","status":"fresh"},"measurement_id":"bffc17539f3443c97cf8012594e6c22586fbc437fd27467abbb4ee17995b403a","metric_details":{"accuracy_stderr":1.58,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:07:14.05849+00:00","total_cost_usd":3950.9,"total_tokens":5760922727},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"OpenAI","url":"https://openai.com"},"agent_url":"https://openai.com/codex/","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"b8d9ccae-23a2-485e-b9f1-a7654c79c560","model_release_date":"2026-07-09","model_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","n_trials":370,"source_row_date":"2026-07-13"},"run_count":5,"scaffold":"Codex","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"b9bf808dec50df0e64e3025aba0c983c695bea80fdf2f962997e816e5bc5d50a","result":{"confidence_high":45.944737,"confidence_low":24.756276,"sample_count":74,"score":34.59,"score_display":"34.59","source_stated_rank":2},"run_fingerprint":"d969f0974e04e7354172c44eb11514df49b6c9e5d131fa248e147ad8d3e0a59e","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":75,"at":"2026-07-22","basis":"evaluation_at","evaluated_at":"2026-07-22","first_observed_at":"2026-08-28T07:09:05Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f669743d1a7a2f1fcb545562605d15f8042ed1a064c3c2d0bbdf6891f68c1d99","metric_details":{"cost":null,"pass_2":37.28522336769759,"pass_3":31.958762886597935,"pass_4":27.835051546391753},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"gpt-5-6-sol_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"330a2d901d6208787dd59902438b274fd530133a84c8829a19757d7db2052656","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.90721649484536,"score_display":"46.9","source_stated_rank":null},"run_fingerprint":"f9f22866810ada5296ade20260082bd7787dbf2fc28629bad2b894f82eede290","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7232eaf7f62d0c1e598509205027fe208290d7c7f94ef12d0168e194ece9d17c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (max) · tauBanking"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol","upstream_model_label":"GPT-5.6 Sol (max)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"23ade084fc38f7474bf05c58f2c4fef336a2917680c322d09980491a27aefbde","result":{"confidence_high":54.244374,"confidence_low":34.84743,"sample_count":97,"score":44.329897,"score_display":"44.3","source_stated_rank":null},"run_fingerprint":"583b249a195123635fe98bdd0babd8b221210898e360287e27a0aa21bc0ebd02","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (max) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ac6151dc0810716b31530fd58fa4713aca3241e9ff0ec48318e13ddfe2442c59","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · tauBanking"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-xhigh","upstream_model_label":"GPT-5.6 Sol (xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"d7c607f6afe55efef2a9af77196bb4d06fc31f38a572711a3d3ac7ec481e53eb","result":{"confidence_high":48.08744,"confidence_low":29.104512,"sample_count":97,"score":38.14433,"score_display":"38.1","source_stated_rank":null},"run_fingerprint":"6b3933d1f4fbedf0608b9c549382afe4c676ced1545c0cc5787eccbfe3dabd41","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (xhigh) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ffd8190bdd3a441c8430f266d3aa8ed91b4e9b441c4afef215e13aacc35c68d9","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (high) · tauBanking"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-high","upstream_model_label":"GPT-5.6 Sol (high)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"5c2f23884e76556fe439a55cff6d64eeb30cb4ff9659acde33526e69151671b2","result":{"confidence_high":46.62878,"confidence_low":27.786541,"sample_count":97,"score":36.701031,"score_display":"36.7","source_stated_rank":null},"run_fingerprint":"1b9a17180551bc348eb2bffe3f52fe92aa8bf58974fe1395c72ea8429bd63a61","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (high) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aa58b7b4d95f089cffb0e742eab51a7b2f4486e959686e46d79f561f9ae3f799","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (medium) · tauBanking"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-medium","upstream_model_label":"GPT-5.6 Sol (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"16cf82b08662f17a589cf9287ffdc1f7509c5ba43516fed314fd40a7e04a03cc","result":{"confidence_high":46.419695,"confidence_low":27.598964,"sample_count":97,"score":36.494845,"score_display":"36.5","source_stated_rank":null},"run_fingerprint":"3f983b5361a18559924cba1c675c63f429e7f6da750c59c9253a3497dc48972f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (medium) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8ac19d029d42b4f6d375f6c580977d997a71835d39108699106c6a92466d84a6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (low) · tauBanking"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-low","upstream_model_label":"GPT-5.6 Sol (low)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"297f87ff5f091985e014411dbe1f56df9d5eec9030f861ae0a70ee21169fe994","result":{"confidence_high":38.768255,"confidence_low":20.970583,"sample_count":97,"score":29.072165,"score_display":"29.1","source_stated_rank":null},"run_fingerprint":"bad58ef31f0e954b8e325e590895c63071da0241259299c543507e91c91ac3d5","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (low) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cbaeb978710570f8fbcb93070f9a5855d4b4435ad9ea8e5b1484f77a496e2034","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · tauBanking"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-sol-non-reasoning","upstream_model_label":"GPT-5.6 Sol (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"146d3141362b5e4230a4e3dd1d10bac590c67897e7d0538c21ba334ba66983ac","result":{"confidence_high":28.57856,"confidence_low":12.91384,"sample_count":97,"score":19.587629,"score_display":"19.6","source_stated_rank":null},"run_fingerprint":"565aede939cc73d50eb14007833dd2bbc2c45c4f2957cc13c26a36658ecda663","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":81,"at":"2026-07-15T19:04:43Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-15T19:04:43Z","status":"fresh"},"measurement_id":"ca197b317697cd89e6a0eb260442a71f254734869eb951e8e4a5c984966e2386","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.4,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gpt-5.6 (sol); createdAt=2026-07-15T19:04:43Z","source_row_created_at":"2026-07-15T19:04:43Z","task_pass_coverage_threshold_percent":75},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"gpt-5.6 (sol)","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"0c27a3d258eee8527a47c4f11eb6374a83f2b73772e95db661f3a01c330d72f9","result":{"confidence_high":84.2,"confidence_low":79.39999999999999,"sample_count":1000,"score":81.8,"score_display":"81.8","source_stated_rank":2},"run_fingerprint":"bf487e31201b0c90272f47685068f5b47975eb27f6f9d12ff6e1636b79dfee99","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=gpt-5.6 (sol); createdAt=2026-07-15T19:04:43Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7bcb5fa02e3e66eaa72b4761","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2b212ebcb1e1ad31de25426e2dd7ed24324306e9652c78e3c196ceb5f9f8e5e0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.652","mean_standard_error":"4.8","model_release_date":"2026-07-09","notes":null,"standard_error_points":550.0,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"service_tier":"pro","source_display_name":"GPT-5.6 Sol","source_effort":null,"source_model_version":"gpt-5.6-sol_promax","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"590705e26c75b45da27aa80c801143a4fab69fd04018826606297123a2e47d6d","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":51.4,"score_display":"51.4","source_stated_rank":22},"run_fingerprint":"65b29396bde2a70f29dc6b5597723ab515e2273370d6a679271b47cfca03a5ab","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"d6f6c91d3072562a44f008898ed0e7670a705afbe24227f7941a416058243c12","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.543","mean_standard_error":"3.3","model_release_date":"2026-07-09","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.9,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5.6 Sol (xHigh)","source_effort":null,"source_model_version":"gpt-5.6-sol_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"beadf7fe680b9c3ab50504aa77d3b2d518b6fe08899e935912955075af37505c","result":{"confidence_high":45.344,"confidence_low":30.056000000000004,"sample_count":480,"score":37.7,"score_display":"37.7","source_stated_rank":14},"run_fingerprint":"27c981a8fbf1ca8edb168d18c9785599e11ceb11e78a8ef496914c68c68a44ab","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"520ad6b8660cb339eae5c86faa868d23c3b7cf6a1f426144e4d1a0149f371f14","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.543","mean_standard_error":"3.3","model_release_date":"2026-07-09","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.9,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (xHigh)","source_effort":null,"source_model_version":"gpt-5.6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"9a67c2a0cad6e604b9b83abe4a218ce6c609d7a0e96fa91e56225e12158d44b7","result":{"confidence_high":45.344,"confidence_low":30.056000000000004,"sample_count":480,"score":37.7,"score_display":"37.7","source_stated_rank":14},"run_fingerprint":"25dd20218d5d49c4aede0b0c6ea56e08367a5abe997ae78225a08635c60d1867","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"068d4f4b1f1fb852794cadfe5c89b19fafa5c7da08f75dcd6274dfee71afdcd1","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":35896},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 5.6 Sol (xHigh)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"06bcf73471d2b9f6d5e6bc55fc5622c93e55a533d6660c0467a0041dfc8f1e55","result":{"confidence_high":0.07732457571198534,"confidence_low":0.051676213416725576,"sample_count":4348715,"score":0.06450039456435545,"score_display":"0.0645","source_stated_rank":12},"run_fingerprint":"15a5b077c85a240deafcd86d8e2ba3dbd12a46f0e0cab06b017990b392d5e648","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9d27c101bd0344a4bbecafc314cdb3a110aed795f3e8d8e326d1d19a3997bfb9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol","source_effort":null,"source_model_version":"gpt-5.6-sol_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"8a79dbbf928d612db6c65ea087a14a0a1840500e78ef2f83840fa5bd8ad54c38","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":9619.367999999999,"score_display":"9619.37","source_stated_rank":5},"run_fingerprint":"d8c48511ad6e9cced5f37bcd750f3e1561878c8cef6e4f9a9baefcb72d3b736d","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a6e3048e0b5def1df0f81987","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f0c1452573298e463d1a912df0b436d718054979a538e2f6db91910b29612606","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-21 / +21","source_locator":"Leaderboard models table · GPT-5.6 Sol (Max) · gdpval"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-sol","upstream_model_label":"GPT-5.6 Sol (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"6050270e4bf49f139b182269eb7b8076b49554fa834aaec09ddb8df21461b07f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1587.9,"score_display":"1588","source_stated_rank":null},"run_fingerprint":"57cc0d9065cf1610532dab71aaa8d6941eb04aa312ac703f6c6a33344f83231f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0076065f179a529ebf412f7d4c76cae6ffd453ed3b5d1aea87542d1f797efe95","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · GPT-5.6 Sol (Xhigh) · gdpval"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-sol-xhigh","upstream_model_label":"GPT-5.6 Sol (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"22ac352d713453dfb9479a6011cd3016ceffca166f7ff7952dfeca2da074eaae","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1547.56,"score_display":"1548","source_stated_rank":null},"run_fingerprint":"68136aa2145bb374581e6f3ae094ec2115057583698d77a2d047b302d914b9c6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1b56e52cdc1b51fa09ca0421","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8e98cf005308015c1a081bab67853d1e6d5ec1d9e83741801978e42574512048","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · GPT-5.6 Sol (High) · gdpval"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-sol-high","upstream_model_label":"GPT-5.6 Sol (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"19ad62ee9f957909753caf643cd75723365db2e40796233bc3bd57d217f3d997","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1480.03,"score_display":"1480","source_stated_rank":null},"run_fingerprint":"f650c24cf0d6d7abed834e0fb34e875538294686df687c0a36d32ad5dc46dda1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79938598fdbb1914df59fc3c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cc805d0fe5e1e5c33dcb1de8a29eb258c7c9080ed009f2dd00bef5b221896c7e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · GPT-5.6 Sol (Medium) · gdpval"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-sol-medium","upstream_model_label":"GPT-5.6 Sol (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"0920e7c130a0d433317e8beb1b32f7b72a4d8406597f994cc94253b2d279a127","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1402.95,"score_display":"1403","source_stated_rank":null},"run_fingerprint":"ca620e02a1b8c64d072e330dec23c871e5dfab63fb702a7ec431d861bc7ea756","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":true,"normalized_effort":"medium","profile_id":"profile_05094233109adfe43508046c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6ad44feacd5feb707f1b3fb3cdd824447d5ece980a0f608e58e6cc494db7b631","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · GPT-5.6 Sol (Low) · gdpval"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-sol-low","upstream_model_label":"GPT-5.6 Sol (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"39afbe01d3e837011d9b7449eec57bf9a10e1a3a9394bb87ce17e2256f151bd1","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1289.35,"score_display":"1289","source_stated_rank":null},"run_fingerprint":"1119a5de5cd136eba2269200cebbca8cb9f3ee04d9584b256718b672f117014e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_3413b43ba5d8c2d1556a984c","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5e93be65f6f8c688eea47390c14f69603e4c2b8ac527641b05c7444a6c9863d3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · gdpval"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-sol-non-reasoning","upstream_model_label":"GPT-5.6 Sol (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"800abc20df809ee3aca4a7b17b7dd6f0ccb71e463c0235c08eab5df4ca0a7533","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1225.76,"score_display":"1226","source_stated_rank":null},"run_fingerprint":"553c693c1dfc5c3939efe15f9a5cc5c69bb5af8e04c567fac23fe3de9953cf30","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Sol (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_d8dd21b4fb83de112c3fa9cd","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:40Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:40Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9d91613ff4155cee29236da50b6022b21f5eb8ea6a78efdc348c95e57d5a93b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"openai/gpt-5-6-sol","name":"GPT 5.6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Sol (max)","source_effort":"max","source_model_version":"gpt-5.6-sol_max","source_row_date":null,"source_scaffold":"batch tool","upstream_leaderboard_url":null},"run_count":null,"scaffold":"batch tool","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"596dfeddf4f879c97fcbf0ae4f906947b4af3f10abbe2ec8c0e260ef3c506b87","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.340000000000003,"score_display":"27.3","source_stated_rank":4},"run_fingerprint":"0cb3e438daeecf382777bf30fa67f5891d41f78f2ea3bef734f330268d2d87fc","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_ae20fb1272fc841ce5c8675a","provider_config":{"provider_default_effort":"medium","provider_default_evidence_url":"https://developers.openai.com/api/docs/models/gpt-5.6-sol","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1c2005cd972ecdb4b7f7974aacb7695564976d0a134696ab031c400401ef5acf","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-09"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.6 Terra; model release: 2026-07-09","source_accessibility":"API access","source_model_name":"GPT-5.6 Terra","source_model_release_date":"2026-07-09","source_model_versions":["gpt-5.6-terra_high","gpt-5.6-terra_max","gpt-5.6-terra_medium","gpt-5.6-terra_none","gpt-5.6-terra_unknown","gpt-5.6-terra_xhigh"],"source_reasoning_efforts":["off","medium","high","xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a7517fee4d5f67287bcbfe5155c108cac15622e2d0a6c6690679a4ae9180bfc9","result":{"confidence_high":162.83,"confidence_low":157.1,"sample_count":null,"score":159.79,"score_display":"159.79","source_stated_rank":9},"run_fingerprint":"bafe4c3b41d1dcf7a89c309e2aef4828461915c3ce5c948a06b731978446c760","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5e3ecc9092fbf935153dfa39","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a5129c98039a1674c99ff5c51a5649b4504a4f166c54b24b00685ebb1e3203e9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Max) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra","upstream_model_label":"GPT-5.6 Terra (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"dacb2fc58e4523a03c3f1cdd3466ea48390c69447735fa1c243b612c6a3c733f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.082925,"score_display":"42.1","source_stated_rank":null},"run_fingerprint":"0c9915a4d75bee9e8a5dc90cd05aef27ad9b66cf0e92927c7fadf26fd38d4986","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"20449fc061501e2a5ebd157a2a9b2403ab35770570decf49eecae99be88949b2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-xhigh","upstream_model_label":"GPT-5.6 Terra (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"58e733d082953311ab5cb8c2c2f16c138c92eb258e0fa56c71f331be30bf9b69","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.951337,"score_display":"38.0","source_stated_rank":null},"run_fingerprint":"c1e434bb76f6782693b346537ca3af3087d567b106023abb834e8bf6af4f92ba","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0f06e1b6cf7bb813f8d9fec0cb41b0937843b75b1dee337976d67aebe9cf971a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (High) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-high","upstream_model_label":"GPT-5.6 Terra (High)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"72e944215e4ded3c1bf3d576aba360a6dc4bf6951be13e936dce0ccdde41b711","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.237656,"score_display":"34.2","source_stated_rank":null},"run_fingerprint":"75bc4c2d9fdfc30cdcb572aa3646bbca0abf0fb8048afd8bb7a08e83cdb60775","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (High) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ca787355aa310c2722801a4540a1c0e2de9a53814bae70dff83094953bd55e41","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-medium","upstream_model_label":"GPT-5.6 Terra (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"23c093a63429ab331b7088d758201918544e12d540adc754b33732162b50a785","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.093802,"score_display":"30.1","source_stated_rank":null},"run_fingerprint":"af097a18ac6f6ac04a1e022c9a040c8b3c7850e807dfd05061f203391bb0f87d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"322e572aa009db1c7498b41e252f7b80531f74025093abd672897fbbe567b1c5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Low) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-low","upstream_model_label":"GPT-5.6 Terra (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"f318bed45ae5e08105922f4406e52c4cc35947fee9a106c8900fa104f3071f2a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.496203,"score_display":"27.5","source_stated_rank":null},"run_fingerprint":"905bb7d8dcf45bf989228fec8bf8569a197508fcbd1c3f957a473f0d905ebc1a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"935762824fba2087bfb2a30563e99535f68253170942c83e7a620a28ef940502","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-non-reasoning","upstream_model_label":"GPT-5.6 Terra (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"67c052f1ee9b0abbed4262c989e0d8798c182d033f600c096cd5aec7a7194933","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":20.779701,"score_display":"20.8","source_stated_rank":null},"run_fingerprint":"fb7330d88d2416af4213b6c2d4b1d844454fdb4fd0f87cd6d8068b34003d2e4b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7cef2a7a036ab2ef403128df5692eaa84372d2bd799fd944c3a1f6c2ad8bfb59","metric_details":{"license":"Proprietary","variance":5.055830326773141},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1469.8826565738907,"confidence_low":1461.0686304780102,"sample_count":37663,"score":1465.4756435259505,"score_display":"1465","source_stated_rank":51},"run_fingerprint":"9e6ecc2803645311c5e2afc9f964e370fa1be45b1e3c83cba7b6e138234ec50c","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"400c911b624bf0d005bc2c6e5a547a5447cc6f97ba9011c79f51d81a079864c7","metric_details":{"category_scores":{"Agentic Coding":54.949666666666666,"Coding":78.24549999999999,"Data Analysis":79.30666666666667,"IF":64.6165,"Language":82.89266666666667,"Mathematics":94.90825000000001,"Reasoning":90.6345}},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.93625,"score_display":"77.94","source_stated_rank":21},"run_fingerprint":"1604e757388101ab96f7b3f1c274cbb8b0cfe38db7f800dd2c73278a3b23ad88","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c1790275fa6ec8f606072a939cf1bf5641b27074daffab4f5cfd6ad14ebf71f8","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Max) · hle"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra","upstream_model_label":"GPT-5.6 Terra (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f7adfb17a9743748c1cfad9e62b6e45b04e2e531290d3c3e321aa6b11a416d23","result":{"confidence_high":45.009167,"confidence_low":40.836234,"sample_count":2158,"score":42.910102,"score_display":"42.9","source_stated_rank":null},"run_fingerprint":"be250ec603a4198f779c31761e1663d039d91a2771a0b2a8d2c2767d97353d1f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b5f2d68de0ba1fa88d493a138610f2f9238b5328e60482717a0d807758d10cfb","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · hle"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-xhigh","upstream_model_label":"GPT-5.6 Terra (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ae32e5fa5ad984e230a29341ced439aa2985da79e4f3a2cb1992d37e9bcc180c","result":{"confidence_high":43.984919,"confidence_low":39.82518,"sample_count":2158,"score":41.890639,"score_display":"41.9","source_stated_rank":null},"run_fingerprint":"3303a2e45edf263061025d8d6e820628f0bc04f61b6545db0b436d4a0ee8d2ef","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"89dc2603897e7debe066918d1ce41d00016f2fdccb9664f502401d0c83b1de15","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (High) · hle"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-high","upstream_model_label":"GPT-5.6 Terra (High)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"361c38a17010c8085a824cbf1c20f98f6405172382f3f80e7731a63eb3f919cd","result":{"confidence_high":40.579699,"confidence_low":36.4769,"sample_count":2158,"score":38.507878,"score_display":"38.5","source_stated_rank":null},"run_fingerprint":"802c2fe18f4b8e361b2677d1bf15a4894f628655970a781d47d4db6d2360724b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (High) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bdd001e390318746376eacc6053cf78fa097ae3c118751c76fcdabaed4b45833","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · hle"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-medium","upstream_model_label":"GPT-5.6 Terra (Medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1147eaf667531853b3d2c549d958d7c5555417073f1c22347de5922dab73dfa7","result":{"confidence_high":35.287757,"confidence_low":31.314792,"sample_count":2158,"score":33.271548,"score_display":"33.3","source_stated_rank":null},"run_fingerprint":"7e6b6b6e5208a4a1dfee894899a5eb77cbebd579d12a697b650172b1591b2296","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"473a3f41df8e06560e2186798cb16fa9a1d054029796fa44251a79afcf803f7d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Low) · hle"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-low","upstream_model_label":"GPT-5.6 Terra (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"13f88aadcb8592055461b3a5c7e3743de155b61d8cbedd096f32904a5bb8fb8f","result":{"confidence_high":31.147597,"confidence_low":27.313744,"sample_count":2158,"score":29.193698,"score_display":"29.2","source_stated_rank":null},"run_fingerprint":"cc16b3ed3e5ef203b0fcbeea4560e09588f21738102ed8a6b583fbc14ae196fc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"decd93b397e6b583efefd6f77695e3af49587fdd47fa766e0989a95affe4b273","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · hle"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-non-reasoning","upstream_model_label":"GPT-5.6 Terra (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"79c654cf04ed7ad9a35547cdeadfef33fafcd75e552d258153429cb2bba1dce8","result":{"confidence_high":12.809482,"confidence_low":10.126593,"sample_count":2158,"score":11.399444,"score_display":"11.4","source_stated_rank":null},"run_fingerprint":"69d32b4408b64e7fcfd781ccf4cdc35d8a6f395613033c45d64d0a14ee32b9fc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T23:44:21.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T23:44:21.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e4b9e951716ab2f27f90102d72b61e0ab6b2e38cef361e1bd083d69df505541b","metric_details":{"best_score_across_scorers":"0.432","model_release_date":"2026-07-09","stderr":1.5672320237336324},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"DienatPbgfgn3EzM7Gvzxr","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"cf9b8ad33e4f2c0dd59dcc48ff4cf1e8b142dea609b0caac629012897ecc5c0d","result":{"confidence_high":46.27177476651792,"confidence_low":40.128225233482084,"sample_count":1000,"score":43.2,"score_display":"43.2","source_stated_rank":41},"run_fingerprint":"6e2092f0ba360ba455cb7c6a61f237493d2cc7db706e1be1f4ee4bd309cf4bcf","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-23T03:39:57.315478Z","basis":"evaluation_at","evaluated_at":"2026-07-23T03:39:57.315478Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7811fc9fbd09e922ededf353d93f4adaa7e8eacf77e70e0cfaf82958771ac646","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-5.6-terra","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1105740443b485d8447f2884f6c8000a8925f10c74bcc6fd8daae9484875affe","result":{"confidence_high":44.61768878165491,"confidence_low":38.47312908086389,"sample_count":null,"score":41.5454089312594,"score_display":"41.5","source_stated_rank":16},"run_fingerprint":"ba2b816667cd0fb99d9993bc6ad869b18d5704bd29b209b690235203539e162e","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a2e31ab0c7a63dee0918defa","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"78b5b5804b7f9704eac5bbf8edd2cfd25fe8a7065d880b4e939e7f7add21d918","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.033997,"date_is_proxy":true,"latency_seconds":36.764,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.906},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":89.899,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"xhigh","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-terra","zero_shot_accuracy":91.919},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"5de4e37b294f29a27d44cb2847507dd5578c04ff3838a158a991d4db054279df","result":{"confidence_high":94.64476,"confidence_low":87.17324,"sample_count":396,"score":90.909,"score_display":"90.91","source_stated_rank":24},"run_fingerprint":"e1639e9846ca3e7775bd1fd724267eecbe5b70e1b9f891206c811ba768c6cb3e","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b0b2454e2765475acccaab20037099a0462c60608331541d446785fc792c5120","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Max) · gpqa"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra","upstream_model_label":"GPT-5.6 Terra (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c32cc75d5193c69e72347b12b933c56770f7ac9a57d28b992adca9c12fe6659c","result":{"confidence_high":95.433163,"confidence_low":87.998599,"sample_count":198,"score":92.525253,"score_display":"92.5","source_stated_rank":null},"run_fingerprint":"01bf449e8f08a62d9bfce9fbd0c53a239d9b159554399bfa2acf71147e2f4de7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Max) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"581a7684fa170bc6d27ff5f2a9fa25ad8bf1639c956977f9d7520e2c8a9294c5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · gpqa"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-xhigh","upstream_model_label":"GPT-5.6 Terra (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"277de4583c324b12117fa4818f9283b95dd81d65b4e0eba675c3dc5ecaeafec8","result":{"confidence_high":94.092166,"confidence_low":85.970617,"sample_count":198,"score":90.808081,"score_display":"90.8","source_stated_rank":null},"run_fingerprint":"0055d3b9c88ce600164dc41e2e69bea86e2eec92afa747b8dfdaa66a53686ee6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7fc8c98945a3d54b479c7ccd4fa6ec2e16ea4a032d3976dec71db34e2cce2674","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (High) · gpqa"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-high","upstream_model_label":"GPT-5.6 Terra (High)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"e9bc138c728600f089c5525dd49099e2d25915f9a026a8debcd91dbe0739e277","result":{"confidence_high":93.121301,"confidence_low":84.563379,"sample_count":198,"score":89.59596,"score_display":"89.6","source_stated_rank":null},"run_fingerprint":"db6fb7d3c5e80336f30848518ef3924330b1f834a57a2d4eb603cf0dbaa37778","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (High) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7f348d695c38ba24f8fead52285913c4024f82948388c22ea60705c13185a879","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · gpqa"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-medium","upstream_model_label":"GPT-5.6 Terra (Medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"66edf69de9ff2fabba94a9f59f658968b0d0b3624e868b4b5dce5cb26b240278","result":{"confidence_high":91.131587,"confidence_low":81.796888,"sample_count":198,"score":87.171717,"score_display":"87.2","source_stated_rank":null},"run_fingerprint":"4084b7f3ca0ac914b83e6a93507c46b33cce235fe29c630dc249a56dbafafe55","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"51ae7a6155ddee8d15036b1eda60c5984dc29ee1d33e2def46c56250aac3a267","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Low) · gpqa"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-low","upstream_model_label":"GPT-5.6 Terra (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8033c431330fae081245cf329f206eb7a8a120639d6b1895dface586bcc0eea8","result":{"confidence_high":88.745528,"confidence_low":78.634041,"sample_count":198,"score":84.343434,"score_display":"84.3","source_stated_rank":null},"run_fingerprint":"26540009251b0ecf017ee8bc4874fcafa02c30e76ef83c829d42d1a35bcbd349","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"74d274ff8576a8b6d5eac90a3424649ffbd5b52759f095f85d57da503c8fd362","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · gpqa"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-non-reasoning","upstream_model_label":"GPT-5.6 Terra (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"3e9a902951696c646c024ef5556c6ea1a7dfb258428d7655994c42d058484271","result":{"confidence_high":80.197384,"confidence_low":68.157367,"sample_count":198,"score":74.646465,"score_display":"74.6","source_stated_rank":null},"run_fingerprint":"0dd82039afe538fdd628d2a70b84ee087da215ce5ab61a3a72a40e599de56b86","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":87,"at":"2026-07-09T17:07:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T17:07:54.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a5286c694f261ee858e34960faae5fd760e542c85f4a563bde3e13a40da006c4","metric_details":{"best_score_across_scorers":"0.9330808080808081","model_release_date":"2026-07-09","stderr":1.5363672147634115},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4bLcFNu5zF6YuZf3qVKoHE","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"8c9debfcac33ebf22bfc0b14804cbf894c2376340ed799f271fe15b856665147","result":{"confidence_high":96.31936054901709,"confidence_low":90.29680106714451,"sample_count":null,"score":93.3080808080808,"score_display":"93.3","source_stated_rank":17},"run_fingerprint":"285c755b5be05b25cbdf2580262aebeeae61567fc400b1c542bdba1d0f06e659","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:35:04.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:35:04.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4aeb2c8123f1d7dd7fbae5f8da0ba9d8fa3b6e839b26cf0d5ace78d029c73218","metric_details":{"best_score_across_scorers":"0.8737373737373737","model_release_date":"2026-07-09","stderr":2.3664359402880177},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"JcuoAWhcnngPEZBPmiUyLF","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJcuoAWhcnngPEZBPmiUyLF.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/JcuoAWhcnngPEZBPmiUyLF.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"8af689d67097ffe0d6480614663f8f7f8b004e3b0a2720e517c8d173beac36f1","result":{"confidence_high":92.01195181670188,"confidence_low":82.73552293077286,"sample_count":null,"score":87.37373737373737,"score_display":"87.4","source_stated_rank":71},"run_fingerprint":"1ed3de57318c6525e80acc13e3f57e6602098524a67e48a045e4c9b6362c869b","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:50:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:50:06.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3a61624ea1ae29b82a788754e24bbfc45c5a01e1a32f86c763fe641e4f442ea4","metric_details":{"best_score_across_scorers":"0.7727272727272727","model_release_date":"2026-07-09","stderr":2.9857515673386437},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"F7v3JtKFb27j2YVmsndLM5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FF7v3JtKFb27j2YVmsndLM5.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/F7v3JtKFb27j2YVmsndLM5.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e3d82b872b1eb0a6a6e7757f3c363ad6cb947ee8e5e27e6de93cbf2da451f90b","result":{"confidence_high":83.124800344711,"confidence_low":71.42065420074353,"sample_count":null,"score":77.27272727272727,"score_display":"77.3","source_stated_rank":141},"run_fingerprint":"327765f28d1dcf84562cfb97790042552574797ce17bec72a570712d82796d81","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":81,"at":"2026-07-15T18:56:33.06547Z","basis":"evaluation_at","evaluated_at":"2026-07-15T18:56:33.06547Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"365fa4e30d771450bc50bf4b1ed631d30f3d0eb0b80292558e7af94dd88ab6ec","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-5.6-terra","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f712bb3df5f8f1c4b2483d52252531f7fc324546c3640c7d81161db8d6b5068f","result":{"confidence_high":89.4050526026104,"confidence_low":79.28181608425828,"sample_count":null,"score":84.34343434343434,"score_display":"84.3","source_stated_rank":13},"run_fingerprint":"e17130c73b0e1a3818271545ab6e6d8c4e90b16cdb33f82870e1f448a0abff4e","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b8c4c1fc96e87c54cdc64309","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f2f8eb675d5a4d14d203e84f","split_or_window":"provider-reported-protocol","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"faf2c4652997fa6552b060d3aa6ce4bb1e50cb880752438a73d029e6924a97c9","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Academic · GPQA Diamond"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort and tools not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"b72ce4576adc1201eec7ec882a588d449d9c2f5cda949167eb39a584f0bce769","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.9,"score_display":"92.9","source_stated_rank":null},"run_fingerprint":"a855fa58e806a985bc514fa27e20cbd04c5bdb8d12d85b2f182f47ad4c41fb5c","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Academic · GPQA Diamond","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c6950733217e3e994d1cd41d","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"157a8374ecb65c2d65ba848dba9c705458134371255dfffd09cf739b2a57f19f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.008039,"date_is_proxy":true,"latency_seconds":8.173,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.333},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-terra"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a41d1c294b13b6e7bb08c68b9a249d2339155cb7b40186318c7a1327371878eb","result":{"confidence_high":87.31168000000001,"confidence_low":86.00632,"sample_count":null,"score":86.659,"score_display":"86.7","source_stated_rank":40},"run_fingerprint":"a896e399c8080978652465e9172e31316c5a1d3ea4725b683d81cf5297356d0b","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":87,"at":"2026-07-09T17:07:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T17:07:54.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f8188d29989add6ac30851c615dc9dcdc1d156954bc3b2cb6d1a0647e68b704b","metric_details":{"best_score_across_scorers":"0.9972222222222222","model_release_date":"2026-07-09","stderr":0.2777777777777778},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jUmmCNezoW7jWiK72SVCaP","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"be69afbe31d7935060cfd908634b1f0b12578c8486f2dfc26e2a3cb0cf474579","result":{"confidence_high":100.0,"confidence_low":99.17777777777778,"sample_count":45,"score":99.72222222222223,"score_display":"99.7","source_stated_rank":12},"run_fingerprint":"a3dd2a076db54e9b59fdb8599b53b52ce9742a91b9147b7e7985d3a5c9c60c40","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:20:21.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:20:21.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fac757375e64044ff7125229a3aa0a6efac7115052daf6af7efe9d2221ae9a3c","metric_details":{"best_score_across_scorers":"0.8888888888888888","model_release_date":"2026-07-09","stderr":4.737793696791344},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Z6S3nMC2tnkpKUixCPLiDj","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ6S3nMC2tnkpKUixCPLiDj.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Z6S3nMC2tnkpKUixCPLiDj.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"29dab9bcbf5f3a08141c9d757f0e8f8673cae956f81fb9189e2bf1ae7bf59230","result":{"confidence_high":98.17496453459992,"confidence_low":79.60281324317785,"sample_count":45,"score":88.88888888888889,"score_display":"88.9","source_stated_rank":71},"run_fingerprint":"e5f484da5a0dbdfc6c0d2bb0269baab92dfc47703d80854d9bb52714a8672c6a","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:42:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:42:38.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5798e1c2085bf22ba51e075c71e4bebc21f9964d897ce011bc56a36cc8d45340","metric_details":{"best_score_across_scorers":"0.5333333333333333","model_release_date":"2026-07-09","stderr":7.521014330903548},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"cjUVqF2qQJ4qhFKBU5BQpW","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcjUVqF2qQJ4qhFKBU5BQpW.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/cjUVqF2qQJ4qhFKBU5BQpW.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"fbef62562d700a4a6f7d6f7af4ee9dcf504dd31ec175882246ce60d22e1638b7","result":{"confidence_high":68.0745214219043,"confidence_low":38.59214524476238,"sample_count":45,"score":53.333333333333336,"score_display":"53.3","source_stated_rank":180},"run_fingerprint":"245f26b9c7a80ba5eb600204864a36e179533d8f83d53bd74775258cf280d094","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":87,"at":"2026-07-09T15:58:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T15:58:30.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ecfd395f960f0b85dc9d13d6ac35ce5ad58477a4d75b2d0817dfe534ac7c68c5","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.8596491228070176","leaderboard_private_problem_count":285,"model_release_date":"2026-07-09","public_example_count":10,"stderr":2.0611471958164294},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"J6evjTMgHkrun8txnEvzdN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJ6evjTMgHkrun8txnEvzdN.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/J6evjTMgHkrun8txnEvzdN.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.6 Terra (max)","thinking":null,"upstream_model_id":"gpt-5.6-terra_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"133ae6c889e89aa9f144f7637d831b8e5ea0fd03397f291fb8ac45efa2691f0a","result":{"confidence_high":90.00476078450195,"confidence_low":81.92506377690155,"sample_count":285,"score":85.96491228070175,"score_display":"86.0","source_stated_rank":10},"run_fingerprint":"38816f0935184f6b37c5a27c1b61d706b90e49de2018931f886df90bddf08d49","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"86698972ce8354f158e08dc263925731ee95f9e921385e03e195728d8de6be14","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.09,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-terra"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-terra","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-terra-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5cbbd56fb7c7da82fcc952237dde82ddda0e41bb8f2be7e491076edadb0ba62a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.89999999999999,"score_display":"84","source_stated_rank":34},"run_fingerprint":"b12a4e6c67011f67c4b8d402905e8ee75740c2ba9ab6364f10b0015ac5f2f8e1","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"29ea8afed421c7b7c6a02be1aba8fdea87d147842862e93942b367982801c51e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.69,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-terra"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-terra","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-terra-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"265c1ee91ea5c967920fdc038d4824347ec1ac6a34d4a3227089f84a6a10d49c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.17,"score_display":"74","source_stated_rank":47},"run_fingerprint":"4d9a62c912017759b47b1cf24875d0a594cdfe5cbdcb9743e997bc485fa694e5","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8256521766bb408db56b05c51442b4869f3fac5424301d41018cef8ae1278915","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.55,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-terra"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-terra","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-terra-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b8d8b72a6c764c1bb09b39ea23a42c3c40a3a8b81bea795ccf4f86d0ccb5860d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.08,"score_display":"67","source_stated_rank":62},"run_fingerprint":"9fdc37bcab4640dc5d17cb1c85e1f8306f74c5a7e21113aa9051bb80b0be889f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"019f916644722348caafce7c45143e64be8bb26d40c95eebdc15d4c909b18445","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.27,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-terra"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-terra","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-terra-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"98962d7c736002858e3fae8060780b09cd938cb9f496cc17d6e104a6f711244e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.5,"score_display":"38","source_stated_rank":108},"run_fingerprint":"2f67fbcd026ab2b9ea7019ba67aa1bcb8b2ca2bc14baaf8c20784add4a0bad6d","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d3f4c98251773dc12726240118a608ddae8ef38aff81ca5f70f1803176382287","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.17,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-terra"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-terra","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-terra-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"201b9ffc948517c99dd86d17278cfcfc10d908585d11edf0d8f7e7f1af8fbfe3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.75,"score_display":"19","source_stated_rank":130},"run_fingerprint":"fec394c7495e99ef532fc862f6644ca2bd1e619b55ee81439bcd8f19fdf725d1","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"133fe9e3f4fcc53456430977d2a4db01eaa941931ac5eb2afb5f3a7e1c06d835","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.09","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (Max)","source_effort":null,"source_model_version":"gpt-5.6-terra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c73b03c13a782e1e5e0cac855360063dde141ac6c116acbc92cac477aabf3aa8","result":{"confidence_high":87.33553,"confidence_low":79.748607,"sample_count":360,"score":83.89999999999999,"score_display":"83.9","source_stated_rank":36},"run_fingerprint":"649a140522ad0bd2f2d7c16fc9ffa6f1fcac9a5809ee3a4e3074ea5153ee6eb2","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3ab11df0951e9b8dafd636c43b69385ca22587706f0b025100df700db82e76e2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.69","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (xHigh)","source_effort":null,"source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"af7b711fea48cdad2af6edc60271a142c15b46c96e94f6725269fa4a5cde8058","result":{"confidence_high":78.419589,"confidence_low":69.410016,"sample_count":360,"score":74.17,"score_display":"74.2","source_stated_rank":49},"run_fingerprint":"8a79cd3c33163101ea4841ae8571d1938d39de9cb53dab5ed0e451196aa06d55","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3035b0f653d73c77d500a9fa54a178a9eb2bc450e62b12e721c621ad693cb9b2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (XHigh)","source_effort":null,"source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0e4c95577dadf149fa0edbe2c21fb8664f53a062f955b8c6c280c02f4e4939c7","result":{"confidence_high":78.419589,"confidence_low":69.410016,"sample_count":360,"score":74.17,"score_display":"74.2","source_stated_rank":49},"run_fingerprint":"20da6c0b458cee1e5fec78e4dfc2ed196eac61bdc1aac5383b225966afb6934a","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4d54c60d18c8259c1d5dfcdcc8f69746ee6ca5ab3b4def7af81ef20483977849","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.55","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (High)","source_effort":null,"source_model_version":"gpt-5.6-terra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ebe61cefdae935f8b0c24bbb228a4008964cc44d8b6264b76e2c2edfef332f34","result":{"confidence_high":71.731685,"confidence_low":62.067639,"sample_count":360,"score":67.08,"score_display":"67.1","source_stated_rank":66},"run_fingerprint":"3925691bdd3f9ec3167d4d4664dfab171e5bdfe90fafe12b6730fc65ad5ef309","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4a8c0c092a3ac809bac1a4f5f2404c5315eff4514d4c0458d4f2e9479ea4274d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.27","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (Medium)","source_effort":null,"source_model_version":"gpt-5.6-terra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5e53ac2a39c28d2e4fb645a302768d3894e3763852db1a4ad50dd095398c6043","result":{"confidence_high":42.608301,"confidence_low":32.65566,"sample_count":360,"score":37.5,"score_display":"37.5","source_stated_rank":109},"run_fingerprint":"08a4819a24c728fb6a43e7c9893c138d202aaec0a4f3fe3d8dd02e59b94178e1","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"348e856eb9f72522b9b0e469caa42d8588395d024102a92a562bcbfeda21a6c4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.17","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (Low)","source_effort":null,"source_model_version":"gpt-5.6-terra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d55fdf913b640281ec257096d6f48ae5b9a74c33897af65cb244962c4ba4cb71","result":{"confidence_high":23.104127,"confidence_low":15.055775,"sample_count":360,"score":18.75,"score_display":"18.8","source_stated_rank":127},"run_fingerprint":"ff658797b0059c0f54919615ce1b47fe7e3a50858767f45d3ae19fcfe806d5b0","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":87,"at":"2026-07-09T15:58:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T15:58:30.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a60630de1d4dd41aa2201ad1679829ba35825914015608dfeb241c71de95b226","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.7073170731707317","leaderboard_private_problem_count":41,"model_release_date":"2026-07-09","public_example_count":2,"stderr":7.194088392074452},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"ZUygpkXqg3NjchXXfedAWV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZUygpkXqg3NjchXXfedAWV.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZUygpkXqg3NjchXXfedAWV.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.6 Terra (max)","thinking":null,"upstream_model_id":"gpt-5.6-terra_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"a24bdb9bd60f8442edd874a72bb0fb71928186fd85d815b59d6bd044216d937d","result":{"confidence_high":84.8321205655391,"confidence_low":56.63129406860725,"sample_count":41,"score":70.73170731707317,"score_display":"70.7","source_stated_rank":17},"run_fingerprint":"1c23817265059d115c958ef186f1e39d0578bf9420606096281f25c205513a8a","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"195ac509e1917152c961cddd982e79c87536ff01f216bca40c15a8e42ad242fe","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.016375,"date_is_proxy":true,"latency_seconds":13.033,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.822},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-terra"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f1ce5c60efbe08e410e65f59466ee576ed34aad250a4fe5555a3fab15b8b52a9","result":{"confidence_high":88.08512,"confidence_low":84.86288,"sample_count":null,"score":86.474,"score_display":"86.5","source_stated_rank":20},"run_fingerprint":"599032d324bd14c3110185fa53be384ae3c81dc41a8f4b28103bfd3308740983","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2ce2956046cbc42520aac5793d2fc5df7160394618b7b051b7f86efd283047c5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Max) · mmmuPro"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra","upstream_model_label":"GPT-5.6 Terra (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4fa3739e878169e08dde308cad2d958fa51b72220a5f30ff168829f23255d207","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.693642,"score_display":"80.7","source_stated_rank":null},"run_fingerprint":"eb6ce26963175df6b05abb64c1462a98c1694aa793bb63967ef66b9f0bb91ee5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Max) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"452eca18fcaa8c9c270b4d630e11f1faeda222d6a1b174df0b0fd361de8e8e3e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · mmmuPro"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-xhigh","upstream_model_label":"GPT-5.6 Terra (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6467515efa39bc8c07603484954ed563bdb856df880464b43928b4e9e96b961d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.479769,"score_display":"79.5","source_stated_rank":null},"run_fingerprint":"2d4b1cb08e533f8d42d001f7c46a7c5f449ebadde9f16bcf891c045662095d6b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d1f042cda95497e12836cd558eea13e72b4e43328f71f6c29e6318efc1213b9f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (High) · mmmuPro"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-high","upstream_model_label":"GPT-5.6 Terra (High)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8fb78dd088560038ec69dd97fcb74ef52719eca7bd28d97a174367900fe4e4d1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.075145,"score_display":"79.1","source_stated_rank":null},"run_fingerprint":"032ddf31618955598f9339e36e23125d085d85ea7c21c6036267cf711de9684a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (High) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6e23c4185db83341f35e8f5c08a8b1fd9913e0a2a4aecd7bb794a2e3eb361369","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · mmmuPro"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-medium","upstream_model_label":"GPT-5.6 Terra (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b2e0990315bed9727f91dc7622e14f1d00e6975ec3047780eb81fd60e1cf8ea0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.763006,"score_display":"76.8","source_stated_rank":null},"run_fingerprint":"9f4bbf65c152d6876abcd261086541f02a6c8c9cac042c782f187e762d5a17e2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4365a95575488a90e4add26b38b615d0f906e841d07695dbb7842de19097426d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Low) · mmmuPro"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-low","upstream_model_label":"GPT-5.6 Terra (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0273579a8b644f954a4fa81e6f01c9f7606ebaaf84dea52901d9a9995996d54a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.127168,"score_display":"76.1","source_stated_rank":null},"run_fingerprint":"dd55d2ef60188b368f314749d038537f0fbe1bf487a84313f8153e6733568ab1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b649735cf166311403bd0c9c7f4189170a45bd0c82f8ac6f485e94f1e98d329f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · mmmuPro"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-non-reasoning","upstream_model_label":"GPT-5.6 Terra (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7c3a4175c83d57e2d6ca07cf9f554a1afc07203fb3e64e7555a03928a16b8918","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.705202,"score_display":"66.7","source_stated_rank":null},"run_fingerprint":"3dad3201508d807db266db8379cfef79b56ffd332c51ebdc0e18c9bded009ada","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_39a5bff9a48a29bea5ce760d","split_or_window":"standard-no-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"4ed019c63c349b86f04542f5d9c7c47c09151222548f3788fe5ac60b89704bde","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Multimodal · MMMU Pro"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7668a8bdde63b0a1b008fd1140ee89a7608afe0b024f589d244f127d83c35ea3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.7,"score_display":"80.7","source_stated_rank":null},"run_fingerprint":"4da5ea8ae1e6fc98ad1129fea31ce2111e7a3230d84a8956c7da56dc06f4442c","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Multimodal · MMMU Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c6950733217e3e994d1cd41d","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_aa0096763ba73cd85e46674d","split_or_window":"standard-with-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"9f0a0affb39943fe9e99f67c575a71d4668f5d48c0ad271b9b2020c528288a9c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Multimodal · MMMU Pro"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"tools enabled"},"run_count":null,"scaffold":null,"tools_allowed":"tools enabled"},"protocol_fingerprint":"c9869f62f1ab1e482460e86f9c0ac73683c41febdcda4cd23712ea1098430062","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.0,"score_display":"82.0","source_stated_rank":null},"run_fingerprint":"9680fa1285439efb8b2f38c9022ed7ad4ce641c49a0fef6df94f9d7a175e2d08","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Multimodal · MMMU Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c6950733217e3e994d1cd41d","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"75dc367a6e153fe8526a0697af1a17b1bee6c7328cacbbc8adf0fca70c2f1c9f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (max)","source_effort":null,"source_model_version":"gpt-5.6-terra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"018f6f30b5408650879c04ac38cd9603320419b1b7778cdea6704d3c82bf0992","result":{"confidence_high":41.45954,"confidence_low":20.593649,"sample_count":71,"score":30.0,"score_display":"30.0","source_stated_rank":15},"run_fingerprint":"44143600eb9efd3d5de774c1bba2c0d9acdb5efb4166ec86088d2876ea057a09","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2e4fa0d115b85ebe2884cfc8e68eb6036e966dc599917aeab8b4ca397fcf5e4a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (xhigh)","source_effort":null,"source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"1532f01299b4b47e0d8c781c5faf01b76371ef3079ca74b656211a9f321feeff","result":{"confidence_high":38.459269,"confidence_low":18.172948,"sample_count":71,"score":27.142857142857103,"score_display":"27.1","source_stated_rank":31},"run_fingerprint":"5f176a603ac1de9f715c1e5fc422161734e974f56c7d7c9aa4ea481d5bd23132","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3d203537fcdea73c3ac570f8b17770be0723cba0eaf02f9a17bc6f53ef14fabb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (high)","source_effort":null,"source_model_version":"gpt-5.6-terra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"2c8800a9de67cf3147cc8bfbe3850c88be4a7f6773c6c192a4336ee6b2edb379","result":{"confidence_high":33.865422,"confidence_low":14.635335,"sample_count":71,"score":22.8571428571429,"score_display":"22.9","source_stated_rank":46},"run_fingerprint":"15da779da0512870e8fdc7c6ad773cc5e2c1df4f163043859dfd61c1328f7cd8","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0add57490f9657bf8bea0b307bbe4129407c353ee1c6a478c838d3873facc0c5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (medium)","source_effort":null,"source_model_version":"gpt-5.6-terra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"30dd3ce282d30a2a0643197c974402fb4316f1f4ae244edb416b6de36d55aa00","result":{"confidence_high":27.856248,"confidence_low":10.34466,"sample_count":71,"score":17.4285714285714,"score_display":"17.4","source_stated_rank":67},"run_fingerprint":"266fa54646ae427eb1ad79d226c5a4ae880e28eefe93dc7d0658b9cbe8d65214","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c945cdb849c47ddec66333b7265c535fbf3f760546c1e3ee22ab4f880258976a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (low)","source_effort":null,"source_model_version":"gpt-5.6-terra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c0c86ec737d7c758b4285714a3242f081b4d6feacdb5f97a173f87cc808fe7d8","result":{"confidence_high":18.451582,"confidence_low":4.570603,"sample_count":71,"score":9.42857142857143,"score_display":"9.4","source_stated_rank":95},"run_fingerprint":"dc41f9e51103b9ee2a4e5e5913193afb33939a31ff14ff42fc454d89ea660d07","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cbb85da75ac7755c76288095b7231c20cdb1f40310e2ee503e334422adf093a5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (Non-reasoning)","source_effort":null,"source_model_version":"gpt-5.6-terra_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6501de5bf8b7787987b9a7e99a037841047f97cb2ddcbea7b9721da2f7a233fa","result":{"confidence_high":8.480179,"confidence_low":0.447477,"sample_count":71,"score":2.0,"score_display":"2.0","source_stated_rank":132},"run_fingerprint":"9deddd32c5049abf948e328119738f0a89acac4cc77b312ec18304a31ed70fc9","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a2ab56d9fb9b8aa4aabb74ce7be67f5d181f38cdc7a6efbf3d1f49a5a98e1db1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Max) · critpt"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra","upstream_model_label":"GPT-5.6 Terra (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8c4d54e83461b57068982929fe53dce736376ee8187c7db996999c5536366f9f","result":{"confidence_high":41.544538,"confidence_low":20.536457,"sample_count":70,"score":30.0,"score_display":"30.0","source_stated_rank":null},"run_fingerprint":"51b723a2e180a8cbfd56a14beb64170cd40d1bf0ef57c8f3e35c4f3a72f3e4b8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"337fcd49de7a2ebfc57f92454dff9531bb0773b733fb92918143d72d102fd1a3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · critpt"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-xhigh","upstream_model_label":"GPT-5.6 Terra (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6dbdc669ac2c8f74d1a719d3e5317858ec21394c4b5c996fff50271949e3ada4","result":{"confidence_high":38.544533,"confidence_low":18.119461,"sample_count":70,"score":27.142857,"score_display":"27.1","source_stated_rank":null},"run_fingerprint":"5e0599b9f4025303649065a47cbfa5166f40da212bc5b627b62fcee985b7a181","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9cd74350980c914138015ad2becdb3df709673c8d14bf4d50c9897d0cc25f6e2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (High) · critpt"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-high","upstream_model_label":"GPT-5.6 Terra (High)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"02e2de6127bbd91bcc4967deb2519f4863b3c5d95102c35acb7a5bc0c56d1d3f","result":{"confidence_high":33.950553,"confidence_low":14.58794,"sample_count":70,"score":22.857143,"score_display":"22.9","source_stated_rank":null},"run_fingerprint":"08a1de81ba21eee7679a21e7bbae953f236364adf8f5bfd51fb1b47d219517d3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (High) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"637784231633bd808acb183d2e085df39ed836224e2b1ad55cc496dd5427ae4c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · critpt"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-medium","upstream_model_label":"GPT-5.6 Terra (Medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a506e7dfe67a2fab505535b34d0edbab7e6ad64fa0a0d17944437ef69ed47158","result":{"confidence_high":27.940154,"confidence_low":10.306037,"sample_count":70,"score":17.428571,"score_display":"17.4","source_stated_rank":null},"run_fingerprint":"3d39a97a89d32d67a50c1b1161a237f47cc8b138c3e4f9d80dfce96dda4c9881","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5f76caafcc44589b9798f33368e4b0e82ac868fe17ecc7ead4ab078e2981160c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Low) · critpt"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-low","upstream_model_label":"GPT-5.6 Terra (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c8fd0445569d4366c3803ef2efe6dfd02301ef97aaa2fb3caef9acd17b3dc2ae","result":{"confidence_high":18.53088,"confidence_low":4.547709,"sample_count":70,"score":9.428571,"score_display":"9.4","source_stated_rank":null},"run_fingerprint":"d6f98b4cd4be93b7615d6e41e4f25577a3c98beb00079557538666f88756676c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aaa89db3e36d3ef45354681495fffba63e551f94c76fa118335e268d68e31d58","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · critpt"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-non-reasoning","upstream_model_label":"GPT-5.6 Terra (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b635679bcbf1018b77727b5be7db87678d52abfe3ede6b8af6cf79d7b4c27fd7","result":{"confidence_high":8.55094,"confidence_low":0.443448,"sample_count":70,"score":2.0,"score_display":"2.0","source_stated_rank":null},"run_fingerprint":"cb4beced8576990d51bfa258b3f0c9fca5024dd92b0ce721630ea90f005a7cb4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d45eb016048992d0c64fd97c446579fb3dc6c83a43542d111e79edae548f500f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"902409fd7a7a4a66284db7f42458a2c436474c8784c97627642233ce61ef4bbb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.9,"score_display":"48.9","source_stated_rank":52},"run_fingerprint":"314f910ad6b1bcd926cfd124ab1ede6001cb2cd4476b0fabd8dccad5ef3124e2","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"affb7efc5873c0d53b9e1a8b5b7070f9b8be7c1ea89a5d1692134a82e08ceea1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-terra_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"4fb8b48bcebc50a68d498a4f2bcec93b849dd68eda3586c8841a4ca751c24c7e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.9,"score_display":"48.9","source_stated_rank":52},"run_fingerprint":"46ea75ab491420a9e19cb73b8ddcf0418eca80822eda4143484ad072069c9fbd","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3ab9f40a739fa78b138cdd9f","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":87,"at":"2026-07-09T17:07:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T17:07:54.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ba1c7570047f235f2cbad1c745ee029e3ef2d36c387697734fcebc2f1ac3cf4c","metric_details":{"best_score_across_scorers":"0.54","model_release_date":"2026-07-09","stderr":5.009082659620331},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CQhcUcXfPPJKuyt57AHZw5","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"431273377c2c8b1459ee8ed95e5bcf18caeed076316ac39d26429a4fb685c66d","result":{"confidence_high":63.81780201285585,"confidence_low":44.18219798714415,"sample_count":100,"score":54.0,"score_display":"54.0","source_stated_rank":9},"run_fingerprint":"844986e0499304754665166a5645331f71db0ef0a31122cf043b9b2d9b93d4c4","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:38:43.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:38:43.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"27cc525f68edfd98960f2f18bc0d554be55f4947b9a9b4e5402268d315c1f079","metric_details":{"best_score_across_scorers":"0.22","model_release_date":"2026-07-09","stderr":4.163331998932265},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"JzDofpJXt6d4f5kbRMSEmw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJzDofpJXt6d4f5kbRMSEmw.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/JzDofpJXt6d4f5kbRMSEmw.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4f78c8c8ac18f409db8a222825785850f94eafa2d9b36dfb78772182a526cbf1","result":{"confidence_high":30.16013071790724,"confidence_low":13.83986928209276,"sample_count":100,"score":22.0,"score_display":"22.0","source_stated_rank":76},"run_fingerprint":"c26f8987f680ea4f093b4845fdbd01ff8bfdfef07a0effed2ff488ea3a5cbae0","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:53:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:53:15.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0e4654695c8d224da8cf8127e4f470890c11fbbd9f0684c6a93aa03273c4483f","metric_details":{"best_score_across_scorers":"0.05","model_release_date":"2026-07-09","stderr":2.1904291355759034},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LmRxqPrS2Gm6Gq2cSNc92G","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLmRxqPrS2Gm6Gq2cSNc92G.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/LmRxqPrS2Gm6Gq2cSNc92G.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6f0fb214748821880a35a3da8fde870f5c5d96ed51073970710f98796013d9f4","result":{"confidence_high":9.293241105728772,"confidence_low":0.7067588942712293,"sample_count":100,"score":5.0,"score_display":"5.0","source_stated_rank":161},"run_fingerprint":"2fde6f322b28ab00d79c0c8000ac023986bed0e66d4e0bb2c7db336186eb1786","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1031bc53845ef66dfbba0a9becf520db73ed078bb8afd2463c8906d882dde485","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.55","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (Max)","source_effort":null,"source_model_version":"gpt-5.6-terra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f49fc5d5dc17d925e46a34e90ac27d33983718b43bdeec786fc58172835b5c89","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":96.5,"score_display":"96.5","source_stated_rank":21},"run_fingerprint":"d0c6ff7f950c67f949ac60b3faf8fd5541e2f49023149951573de66be8f83594","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ec93098cfcd336649f422acd24967e067b73b1635f2146adfb818215c85f2b7b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.26","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (xHigh)","source_effort":null,"source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"61f8695f387b68b4ce897ea895ac13a9024dd684a905d7f7ec67175cf87c3e72","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":94.0,"score_display":"94.0","source_stated_rank":42},"run_fingerprint":"496efb4047431cb625e5700452aa767c819857ab0a9f7777cc63613f2897652e","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"60e7e05723666fcf4469bdc9ed91d5fefe3f0a82a0151007a56568ac41eb9172","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (XHigh)","source_effort":null,"source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"afdf65f0678e1964be31bd3973e644d169aea15ed25e2eb63968b5ab6e686c5d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":94.0,"score_display":"94.0","source_stated_rank":42},"run_fingerprint":"678ca459198e27c74fde7d4b9a44cfe59932de607ac2b9bad4c407ec3b89d57c","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9c9c08b7a7f2f881b458db1887f72a8fc8bb40cb05d0c018f97899c4a7078d84","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.19","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (High)","source_effort":null,"source_model_version":"gpt-5.6-terra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"15cb0a5917b5f5bcc8b6d8044f2e18fa62ae412efb4bbb03e64441b90995c352","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.0,"score_display":"92.0","source_stated_rank":59},"run_fingerprint":"73b4f844482539ac0a302e26d57b5045c787a93454cda1c9bc72e75915d0ab4d","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"461a1286a222db0717bf6218623ad55f951cff9020609d3acad9414cf6ee9327","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.13","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (Medium)","source_effort":null,"source_model_version":"gpt-5.6-terra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"8feb6fd42431857acb58abdfc6e76c9b9b3ad0925e077c23b25619df769d1958","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.0,"score_display":"77.0","source_stated_rank":117},"run_fingerprint":"76482a7789124349f73ea555e846324db73d2363a4c1bfe9d3e5695c4f34a0bb","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4ac1e580c214c04a769e8ff6ba2bc8327c7bdbb91f209d7eb67f9765dada5d5f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.09","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (Low)","source_effort":null,"source_model_version":"gpt-5.6-terra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"359a5a0526b3d186bab631e48d03bb8fb95e28026946c9993aedff1f96f38626","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.17,"score_display":"60.2","source_stated_rank":149},"run_fingerprint":"7571bd8e6ea8447d1d0c5447273870500c6c7bb0aa36c433453fda21a30c2911","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"204a86a745474abe96e19ab592c67cadd882aee85fc42c1860a0f6d1e21e09d0","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.400788,"date_is_proxy":true,"latency_seconds":180.284,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.938},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-terra"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"399598e9eee3b24d45f2bcfa7b20c2d4f95d3239ba45b969c881a98c4944d784","result":{"confidence_high":97.23848000000001,"confidence_low":93.56152,"sample_count":500,"score":95.4,"score_display":"95.4","source_stated_rank":5},"run_fingerprint":"0b84f3a95c7ea74431b5c183103154ac1c5ebc5cac01cc6ae9d7aab66f36843b","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"f8f4813d3f8b1933d5f29a92aaf14ae4f72e9829122d8a04bd22b3d8f0d14437","metric_details":{"ci_attempted":451,"ci_half_points":2.5569389504458404,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":314,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":75.9290465631929,"mean_cache_tokens":8652201.720620843,"mean_cost_usd":4.945847263858093,"mean_duration_seconds":1016.6698429312638,"mean_input_tokens":9230560.541019956,"mean_output_tokens":71938.62527716186,"median_agent_steps":71.0,"median_cost_usd":4.114557,"median_duration_seconds":927.549964,"median_input_tokens":7288778.0,"median_output_tokens":70835.0,"median_output_tokens_to_pass":70061.5,"median_peak_context_tokens":177398.0,"n_attempted":451,"n_passed":314,"n_tasks_attempted":113,"n_tasks_passed_any":100,"pass_at_4_points":88.49557522123894,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_terra_max","source_model_version":"gpt-5-6-terra","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"869d4cd8ddde2355153ef770f54e40b2142f9c58a3a57dcd9757271830347ed1","result":{"confidence_high":72.17999881740815,"confidence_low":67.06612091651647,"sample_count":451,"score":69.62305986696231,"score_display":"69.6","source_stated_rank":14},"run_fingerprint":"bffca5bde24465d231f8a2a3813eb75aaf5da1ed2736e912ccfd7b929ccaad93","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"937f10ec451f8056e3334a45c07c75b09245348673c55afd37c6322e1884d3cf","metric_details":{"ci_attempted":452,"ci_half_points":2.1242972019271256,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":272,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":43.06637168141593,"mean_cache_tokens":2926653.1681415928,"mean_cost_usd":2.127191725663717,"mean_duration_seconds":584.0832282876107,"mean_input_tokens":3246355.4668141594,"mean_output_tokens":39616.53982300885,"median_agent_steps":39.0,"median_cost_usd":1.8670102499999999,"median_duration_seconds":509.04962550000005,"median_input_tokens":2490234.5,"median_output_tokens":38006.5,"median_output_tokens_to_pass":37940.5,"median_peak_context_tokens":108741.0,"n_attempted":452,"n_passed":272,"n_tasks_attempted":113,"n_tasks_passed_any":91,"pass_at_4_points":80.53097345132744,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_terra_xhigh","source_model_version":"gpt-5-6-terra","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"1d4a74590a3043d0ca472e0d51f9a7dc264e90ed1ec5572b9ab42865886dacae","result":{"confidence_high":62.30128835236961,"confidence_low":58.052693948515355,"sample_count":452,"score":60.17699115044248,"score_display":"60.2","source_stated_rank":33},"run_fingerprint":"d3c450c52e1395bf67693176c423102e8d40d3131d01dbf423b39aabbcd4cfee","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"4b499a006d1e76ec98845ccc08c3e0a7ea99afc43d830ae523c54bcd3422959c","metric_details":{"ci_attempted":452,"ci_half_points":4.328970467213552,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":243,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":33.51327433628319,"mean_cache_tokens":1369338.3362831858,"mean_cost_usd":1.1343762765486725,"mean_duration_seconds":364.2859819269911,"mean_input_tokens":1557052.774336283,"mean_output_tokens":21517.03982300885,"median_agent_steps":31.0,"median_cost_usd":1.031709,"median_duration_seconds":314.963429,"median_input_tokens":1249365.5,"median_output_tokens":20865.5,"median_output_tokens_to_pass":20408.0,"median_peak_context_tokens":68750.0,"n_attempted":452,"n_passed":243,"n_tasks_attempted":113,"n_tasks_passed_any":91,"pass_at_4_points":80.53097345132744,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_terra_high","source_model_version":"gpt-5-6-terra","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"f24670c083a28d31eb9ed8e4196cdbd532898454f893383e788b067d6d0dc9ae","result":{"confidence_high":58.09003241411621,"confidence_low":49.432091479689106,"sample_count":452,"score":53.76106194690266,"score_display":"53.8","source_stated_rank":43},"run_fingerprint":"29f16f2b568ba86a11e6e69e79181b8d86b5b046d4367187ee6bda0f8b016de6","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"de46ecf2d05a24ebbb5e5a88d465ad88cf0effe17f5b8d94afdd672e454a7cfc","metric_details":{"ci_attempted":450,"ci_half_points":3.3819643034066322,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":158,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":25.14666666666667,"mean_cache_tokens":624756.0533333333,"mean_cost_usd":0.5832493688888889,"mean_duration_seconds":229.60274282444442,"mean_input_tokens":725100.8355555555,"mean_output_tokens":11746.56,"median_agent_steps":24.0,"median_cost_usd":0.546099,"median_duration_seconds":193.9660975,"median_input_tokens":628495.0,"median_output_tokens":11453.0,"median_output_tokens_to_pass":11448.0,"median_peak_context_tokens":44178.0,"n_attempted":450,"n_passed":158,"n_tasks_attempted":113,"n_tasks_passed_any":68,"pass_at_4_points":60.17699115044248,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_terra_medium","source_model_version":"gpt-5-6-terra","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"08bf1263e61cf13783464384faa09d1e21261f56a0bfab9122f6ec2e99cdc206","result":{"confidence_high":38.49307541451774,"confidence_low":31.729146807704478,"sample_count":450,"score":35.11111111111111,"score_display":"35.1","source_stated_rank":62},"run_fingerprint":"52bef09cd2d52f738ef91821c3670bf87a184e481efc2845be74b834a50c88ef","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"689846f64e9688b03f95b62043040125b3c1043a693c22081428e6f5c9da91a6","metric_details":{"ci_attempted":449,"ci_half_points":0.7767613669447833,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":108,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":21.456570155902003,"mean_cache_tokens":401000.90868596884,"mean_cost_usd":0.4277452605790646,"mean_duration_seconds":176.52513473942093,"mean_input_tokens":480565.34521158127,"mean_output_tokens":8572.26280623608,"median_agent_steps":20.0,"median_cost_usd":0.3989895,"median_duration_seconds":151.960538,"median_input_tokens":405397.0,"median_output_tokens":8265.0,"median_output_tokens_to_pass":8024.5,"median_peak_context_tokens":33373.0,"n_attempted":449,"n_passed":108,"n_tasks_attempted":113,"n_tasks_passed_any":50,"pass_at_4_points":44.24778761061947,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_terra_low","source_model_version":"gpt-5-6-terra","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"6d507378cb3a2910d0af4f31a8bc178aef77fd1b304458fa3aabf387f852be6f","result":{"confidence_high":24.830213482757703,"confidence_low":23.276690748868134,"sample_count":449,"score":24.053452115812917,"score_display":"24.1","source_stated_rank":67},"run_fingerprint":"4fde3c7bb01f49904e9f942ea9a0cc5d28f06044cafb252deb39f0b6263f91b1","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0bcb8efa31aa36bd59239d24347b52cfe1e2eebe09ca2aef8f606bb6299c1a5d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"75.9290465631929","mean_cost_usd":"4.945847263858093","mean_output_tokens":"71938.62527716186","model_release_date":"2026-07-09","notes":null,"pass_4":"0.8849557522123894","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-terra (max)","source_effort":"max","source_model_version":"gpt-5.6-terra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"d13ae75efc0f54b7da1b7ef2c9d21ae316d58f6261901e61cf2c9a5eb21626c5","result":{"confidence_high":72.17999881740815,"confidence_low":67.06612091651647,"sample_count":113,"score":69.62305986696231,"score_display":"69.6","source_stated_rank":14},"run_fingerprint":"8c08f044e89f7fb454f92eb4bfb29024f3e889ee78011da6f6379c8319296aed","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"113bcb316527f43063ab108749610b5437e13bb7520def74dbf2a50c2488a32d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"43.06637168141593","mean_cost_usd":"2.127191725663717","mean_output_tokens":"39616.53982300885","model_release_date":"2026-07-09","notes":null,"pass_4":"0.8053097345132745","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-terra (xhigh)","source_effort":"xhigh","source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"f930ef31bb5e8b2cf73ff51c697fb430b9e245346e400566b8190552510bce39","result":{"confidence_high":62.30128835236961,"confidence_low":58.052693948515355,"sample_count":113,"score":60.17699115044248,"score_display":"60.2","source_stated_rank":32},"run_fingerprint":"d44dab426ca2372c1a36d2bc41a017aaba96127bf29f912a7f86a5978b73e2bc","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a682d35a830ac98d1328e86f935aaf9dd63d04e9aba78a1a0b4b6a84c90ff11a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"33.51327433628319","mean_cost_usd":"1.1343762765486725","mean_output_tokens":"21517.03982300885","model_release_date":"2026-07-09","notes":null,"pass_4":"0.8053097345132745","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-terra (high)","source_effort":"high","source_model_version":"gpt-5.6-terra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"66e488bc5f0c52f1ef0ff552b0446dffe4c26bce48144d8421bf6022b3b5a1f5","result":{"confidence_high":58.09003241411621,"confidence_low":49.432091479689106,"sample_count":113,"score":53.76106194690266,"score_display":"53.8","source_stated_rank":42},"run_fingerprint":"828497ac142c6a1f1ca703645faee0f02f1fe7e51c42d491c59820ac2125294d","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b8591189487fcf70b26153fa32a1a14f4e47cffc75700a1b71747fdc80d0ef21","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"25.14666666666667","mean_cost_usd":"0.5832493688888889","mean_output_tokens":"11746.56","model_release_date":"2026-07-09","notes":null,"pass_4":"0.6017699115044248","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-terra (medium)","source_effort":"medium","source_model_version":"gpt-5.6-terra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"839b0b6bdaf2421b23bd341d8fa8583dbefb101caa38a60cad8336afdf460199","result":{"confidence_high":38.49307541451774,"confidence_low":31.729146807704474,"sample_count":113,"score":35.11111111111111,"score_display":"35.1","source_stated_rank":61},"run_fingerprint":"5f5e1f26f0cc53e24eff744e54611c72c76b93f3daa57a82955c37b6c4a138a4","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fb884a26c3db900aec8eece0684133ddce6338aa33a5c85cffb50f5e76dfa88e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"21.456570155902003","mean_cost_usd":"0.4277452605790646","mean_output_tokens":"8572.26280623608","model_release_date":"2026-07-09","notes":null,"pass_4":"0.4424778761061947","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-terra (low)","source_effort":"low","source_model_version":"gpt-5.6-terra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"ceab1aa1190a756c54fc3dc20e8b3bb046c33d745197f9bd0d6490bc0d06408b","result":{"confidence_high":24.8302134827577,"confidence_low":23.276690748868134,"sample_count":113,"score":24.053452115812917,"score_display":"24.1","source_stated_rank":66},"run_fingerprint":"b9126a8dec412b4ae1f4fb62fcaa375ef0923ea2c9f44570f888a1aa0184f65c","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_e361abe5266a3ddfdcef017b","split_or_window":"openai-provider-comparison","unit":"percent","version":"DeepSWE v1.1 · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"2caef2adb3cc673e9f406f23fe16452563a5230326ed0ea5166bcb820a295b34","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · DeepSWE v1.1"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value for DeepSWE v1.1; competitor rows are secondary re-reports; exact scaffold, task coverage and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"coding-agent tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"28bd89a57c6784aa44dda4f48bb1ba1e048300dc4313759e24d750d50b4e533a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.6,"score_display":"69.6","source_stated_rank":null},"run_fingerprint":"6e890495ea38373f8ffba3f71107e6e93fea4ab6dc22f5f06ba5c8a5a901b4c7","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · DeepSWE v1.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c6950733217e3e994d1cd41d","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_7664765571d731e0f6ff7adb","split_or_window":"openai-provider-comparison","unit":"percent","version":"SWE-bench Pro · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"13a25a7cc1d6bdf593ba780b00dd03358d1360219e5d415201745883fe85f24b","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · SWE-Bench Pro"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; OpenAI rows are provider results, while competitor rows are explicitly retained as secondary re-reports; exact scaffold, task revision and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"bb784ac6361cbbe179556c67a1258b8773e5fdda82afa9a06e9f85b5d8df29c2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.4,"score_display":"63.4","source_stated_rank":null},"run_fingerprint":"b788cfb042650e8dba49e0b56fe605713151ebf67316f07740fecc14472eee4e","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · SWE-Bench Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c6950733217e3e994d1cd41d","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"540d8ff480cda616a852803e5d50793a7a50c4971457454ec68888703f1a754b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"5.14","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"107","tokens_per_task":"60814","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra Max","source_effort":"Max","source_model_version":"gpt-5.6-terra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"bee89921150d4baef9cb6f24b66fbe4758b0d9cf4c6d9a180d85d98a64dd69c9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.3,"score_display":"41.3","source_stated_rank":24},"run_fingerprint":"7049d124ab40ddf6e4a64ba4fb5e1efce0736c12c6e65087f72ddd7ac3ae3de4","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d22d0f83725b3bf367fc65a0b830dd9c474997fc4623fc3c1182f6e776d9e924","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.81","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"43","tokens_per_task":"23436","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra Extra High","source_effort":"Extra High","source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"33c3b6112d8610dda8a1f5b8f04a87c6ef6846d5122d20014ec0e918f91db355","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.6,"score_display":"33.6","source_stated_rank":37},"run_fingerprint":"785ef77d90daba3abfa702d6156d6797113162fa6b0a4acf65c020aaaf46cbae","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f121d079c2190b527508a5fb84998fb5b3203bc271ae52c4aab8ab5b33fcac43","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.11","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"33","tokens_per_task":"13162","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra High","source_effort":"High","source_model_version":"gpt-5.6-terra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"899c5b9fddad2670b5b950094ab3e4b845d92a2329826bd2011004515d0a2409","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.7,"score_display":"30.7","source_stated_rank":46},"run_fingerprint":"385b422b46327cea9a43d83171ee097ff1f408f4382843ed3943e73ff95dc930","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dc646ece6288534055d92b40ac506c875ed31304bd462b06a17bfd96e6e5fe24","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.64","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"25","tokens_per_task":"7307","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra Medium","source_effort":"Medium","source_model_version":"gpt-5.6-terra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"ea95a98b5f682b9bc4def8140f2db6929e22c921a6a2fc1199661eb0d37ea6c1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.6,"score_display":"27.6","source_stated_rank":51},"run_fingerprint":"e5a6e4b980feb987b2f9a6c933c3a794f4ca2af2bd009c894341d5911a1d5709","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ff401bbfd992c755e0f62c77705a99d5ae4be3adaa94f6f039d43b5621ad8d41","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.52","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"23","tokens_per_task":"5914","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra Low","source_effort":"Low","source_model_version":"gpt-5.6-terra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"0191e7285390b49ce8a9ca9b231e477d657910cd078825268e4df5c26b5a4e11","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.2,"score_display":"25.2","source_stated_rank":52},"run_fingerprint":"b3b96557539c1f82a2ca0581c6f1f99eb74bf315af81106f5adfd98193beb07c","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4210dbde01e85a4b04053d53d75971dfa73b250522a639efaa1b9b13de4cfb65","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra","source_effort":"max","source_model_version":"gpt-5.6-terra_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"codex","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"28b116966df3105e70a47466a630f92d7e1cc8b81dd0f1635b10ae1073527658","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":41.31,"score_display":"41.3","source_stated_rank":20},"run_fingerprint":"e4f6838aeec17c80c17704ed85bdd44ad924220d740bd40db79cf63c075d1080","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b9ff576bf62185e7765a7b0793e10bfc0dcae0e3c7c8ea71b0e2923b5cdcf7fc","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":8.664009,"date_is_proxy":true,"latency_seconds":4992.838,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":6.531},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-terra"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"17155804441dd54c1626b893935d82591489581b594ed8fc556d929e4ed891b5","result":{"confidence_high":100.0,"confidence_low":74.81024000000001,"sample_count":null,"score":87.611,"score_display":"87.6","source_stated_rank":7},"run_fingerprint":"2bad46d20d48d6eee32044b115f1b1c903885633562dc9f5404e77e79bf8ae0c","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dd1538ee4427547d7a21be1baa81fb459dcfbe8cb973b7bcf237e03b639c592b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.031103,"date_is_proxy":true,"latency_seconds":44.331,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.017},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-terra"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"43326c2f724542cbae2fe2235a0dd33d781d1b8605875f83e0561d412c96158d","result":{"confidence_high":87.92332,"confidence_low":83.93668000000001,"sample_count":null,"score":85.93,"score_display":"85.9","source_stated_rank":27},"run_fingerprint":"3cf97abf2be83e45434fbb4a926e826b5090a56098ad646d5a835136721b0adf","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5798803962082b61c6fc05d7e278c7dc2e75266aec1b9995d5a0ecfa2c719b77","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (max)","source_effort":null,"source_model_version":"gpt-5.6-terra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"96d09f1a93572c7c6503cfbe9b0394a20fd7a20649a35ff1e05143c13c116709","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.976851851851805,"score_display":"55.0","source_stated_rank":44},"run_fingerprint":"9dc1d67951d2aac51822f4b8c117c08490d7a206208bb46840c587bbea02ad69","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0403e6d1ff32905cad25c90380b0f5e35325d24388d043b9989919d02105d155","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (xhigh)","source_effort":null,"source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"af7055b7151084826b090192469574ea71c7c1e59f1931d957309d73021a81e9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.6203703703704,"score_display":"51.6","source_stated_rank":75},"run_fingerprint":"6e9005d00d0c2d2772590413c6770ab4adb38204356dc28f6f7ff204494a561c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8c0d960898c1905512023994cb08b48a3f74faf1697c4738dd14167aa41832c1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (high)","source_effort":null,"source_model_version":"gpt-5.6-terra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"5aca3c89a8230aa3babb24137a3170f36d87e1e2b0bff8e62ff5b922437daf0b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.1157407407407,"score_display":"50.1","source_stated_rank":90},"run_fingerprint":"2cb04c1eb6fa7443a62fe834734e44088dd4a32cb44fc09b7acbd43b3bce8c27","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4eefd1ac6302f35299c39811e3a005e355ee69dfbbca78b01fff54e32f3f3e5f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (medium)","source_effort":null,"source_model_version":"gpt-5.6-terra_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b7caf53591020b7202a6c9e879bce3de6c561ddce8b76b181cedd1b5306ba086","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.6527777777778,"score_display":"49.7","source_stated_rank":96},"run_fingerprint":"582d970844b6c3d73646e3600b9e7e53f35b198c5df13d33fa2674e683cd2f29","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f97bae28cb23ce48b8483b630c9e7e4a16a70e79bbd635a614ac45d50deb466e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (low)","source_effort":null,"source_model_version":"gpt-5.6-terra_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"03b6c0cd1b629f269be2c90668e97b9670a325b6d7905791c8df75e50b8e6719","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.189814814814795,"score_display":"49.2","source_stated_rank":97},"run_fingerprint":"5aa0d293dcaa9136c9d529a278ecfac8123671ff108de583fcc8ec43f5921984","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"98dc5e978721f4b7b8855395e1e8303016798b6e788771a9e57613b6a2cc0609","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra (Non-reasoning)","source_effort":null,"source_model_version":"gpt-5.6-terra_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"aaade90218b26561671eac4493c70153f92b3bef48dc1d532f017743be55c9b3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.560185185185205,"score_display":"44.6","source_stated_rank":127},"run_fingerprint":"dd011b51304b1b2322da3b7c09627df4d3a0155fa9c2f0b27d35e21be9cb466e","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4a091b36cf480e675e90dfb51a728a6200723c9c4c6316ff76588db98f2a83d3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Max) · scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra","upstream_model_label":"GPT-5.6 Terra (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1b723abe1189ac71070d6acaf665f319b7f415213ad28c126987b86ff48a4c3a","result":{"confidence_high":60.6198,"confidence_low":49.202881,"sample_count":288,"score":54.976852,"score_display":"55.0","source_stated_rank":null},"run_fingerprint":"9a184ed1f534b1060b57e08263ab414b143287b19f6444402bf99995a99060f6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c170c3576fa55b96e080428ba98217c1e42de3b2839d6cd01eaf50c41e632f06","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-xhigh","upstream_model_label":"GPT-5.6 Terra (Xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"89b8ed5935c85569bc7b4c3097cf70318f0c718871fb5d36e51384d474c62b9d","result":{"confidence_high":58.014847,"confidence_low":46.553842,"sample_count":288,"score":52.314815,"score_display":"52.3","source_stated_rank":null},"run_fingerprint":"e23cd717d8435094f2b7eee14f7555f35d42adab3f9c52d3ade3eb71d3f23aad","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"03e7c615da664ce44adb3490bd899a52e4b6ca47d589f504d2d5671dbdb491cb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (High) · scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-high","upstream_model_label":"GPT-5.6 Terra (High)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"099c70965c4a3e60037dbded5543c247e9d54802a434b4f636a542632537e248","result":{"confidence_high":58.128442,"confidence_low":46.668682,"sample_count":288,"score":52.430556,"score_display":"52.4","source_stated_rank":null},"run_fingerprint":"4098ca0a4c41c721a9df8f11ac22b36422349923f5939e88bff607f5661c9fe8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (High) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"58524b690840be86712ca8c651c6d9154ce7cc8126864d7c396b38d103f00a59","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-medium","upstream_model_label":"GPT-5.6 Terra (Medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2eb4243d73c42fee46045c17ba1940d3f96e131905bcd2bc1b2c778d091e1105","result":{"confidence_high":56.193199,"confidence_low":44.720539,"sample_count":288,"score":50.462963,"score_display":"50.5","source_stated_rank":null},"run_fingerprint":"dfd077f13dcfb05948b822757c7149ab0d4e11a75cc1ab8121295b6b14c0b8a7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8c2ffee9b6fea6b15e7e5c63fed12221dfeb130a64085a2eb7ff49903ac7f5c3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Low) · scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-low","upstream_model_label":"GPT-5.6 Terra (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"6eb650b0304242db61832b5646209e24d708c6dd3e9b82d47f74b18f9b52dc97","result":{"confidence_high":55.62234,"confidence_low":44.149225,"sample_count":288,"score":49.884259,"score_display":"49.9","source_stated_rank":null},"run_fingerprint":"71c85217ec2c7218594eba7ecc5e112219462e824510bbbdde9e9139d2042f3a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":19,"at":"2026-09-15T17:15:32Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-15T17:15:32Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7dda70097c1c5bad9e225cc81f56036abb4e2e6df332cb1830b2df88aaf5ecfe","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · scicode"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-non-reasoning","upstream_model_label":"GPT-5.6 Terra (Non-reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"32635388d3bebc88f7dc6322838dba2067744794142229ea4c038922ce417128","result":{"confidence_high":50.912632,"confidence_low":39.493122,"sample_count":288,"score":45.138889,"score_display":"45.1","source_stated_rank":null},"run_fingerprint":"c61dd8ddca2eef4ed1c7c198f7cc0d7edc98bb5aa5070b128394d8c1446bbc3b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f1087660390d1b34903b2097da7a4f180659baeee749a7c32a3e4b1209bd494f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"1.254047","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.6-terra-max","source_effort":null,"source_model_version":"gpt-5.6-terra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"27f0be14ac4f083323244ac5b6ffd62ecd97b3194345e3ac7bd2b981343f7641","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1951.42,"score_display":"1951.42","source_stated_rank":8},"run_fingerprint":"b91192c2c6fbf2120c92ed8de263f123635751ec260c70af5f4b7492edf561a2","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"14ef6b8e458ce2d03514d7bccfd7d8d94396166d72d53eba95d232e5404e15e6","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":7.887015,"date_is_proxy":true,"latency_seconds":1152.433,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.203},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-terra"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"c52c8446247d419b6c2ffebcd30d3370cc2c68c119ecacd30d4f7f5278bb76d7","result":{"confidence_high":82.83188,"confidence_low":66.35611999999999,"sample_count":null,"score":74.594,"score_display":"74.6","source_stated_rank":31},"run_fingerprint":"d0e59ba3e909215f4cfa33cd2003cefa7e877425c37e308ca4a87879b33b83bf","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f8cc8f234cee419b9531100e70459cba6b35a1b3563111bb32682bdf7de58cdd","metric_details":{"license":"Proprietary","variance":11.146189096050435},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.6-terra-xhigh (codex-harness)","subset":"webdev"},"run_count":null,"scaffold":"Codex harness","tools_allowed":null},"protocol_fingerprint":"182712c291d0c749f6f859c061aac142df484df4aab9ab5a0af15bd76a009422","result":{"confidence_high":1525.6320962616437,"confidence_low":1512.5450604750117,"sample_count":12526,"score":1519.0885783683277,"score_display":"1519","source_stated_rank":46},"run_fingerprint":"fb83933ff625326e8be32a4611ebb82718e995c63a68987093064a2bba45fb4c","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ee796b6ccf318ff8a86c37e4f5d49eed74f552c876599dcf948da23a4dec91b5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-terra_xhigh","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"27365706324c7290dfc04711ff529aaa213c9480e0907f78ec996ba4e5a4545a","result":{"confidence_high":1529.0,"confidence_low":1513.49,"sample_count":7694,"score":1521.25,"score_display":"1521.25","source_stated_rank":43},"run_fingerprint":"20adcfd2869f68d4c46a087578fc16466547a440282e529d95de2a5924a87569","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f844a5720e4deb1d93b5800cdc05accaaa734410698e57f020e5b59019983d37","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-terra_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"c3a637f5e3e24eea32adeca84ed74d99473b1db1a5d4d3c763e35c76d785f768","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.27,"score_display":"78.3","source_stated_rank":19},"run_fingerprint":"cbf94457d3e76b0f1459222cb2f8bfd07a2d72fd874819f54a051e3c177449ca","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3deac3384b6708820298d5a502127c12b45e33a04a90faa2e41694dc0359b254","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.472552,"date_is_proxy":true,"latency_seconds":519.739,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.247},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-terra"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"18472fa609f369146d7aae5472c8c30a3d3513f2e74dad2a1fbd0542e53fb31e","result":{"confidence_high":81.93212000000001,"confidence_low":73.12388,"sample_count":null,"score":77.528,"score_display":"77.5","source_stated_rank":14},"run_fingerprint":"213a1238bfbb3698f3c17428c6a380707c0b1a77c35c48ef302d5a982e5d4e6e","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"beca5ff75acc4ea71992ed0b80d7d7d0be498befd2ac45ad434d0ab624887dbf","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Terra (max) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-terra","upstream_model_label":"GPT-5.6 Terra (max)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"6e097aa528cba28ae9b3a8a1f2046edbfa9c1f62e705e2f92b8c1a1c693b837e","result":{"confidence_high":93.233339,"confidence_low":79.650655,"sample_count":89,"score":88.014981,"score_display":"88.0","source_stated_rank":null},"run_fingerprint":"433364c8ba85034ffce71166761d0b5bf2aed4b805bbd35d5c89085a61c74c57","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (max) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5255b0f4987ac57ab9f1938f72ab690e1aa3b477cb8336c14428aa5999312be3","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Terra (xhigh) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-terra-xhigh","upstream_model_label":"GPT-5.6 Terra (xhigh)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"df6e5bdb3aafecdb004c0c0a9e150a2a4b4f219381e22415f24b31e94be0ca99","result":{"confidence_high":87.111311,"confidence_low":70.693237,"sample_count":89,"score":80.149813,"score_display":"80.1","source_stated_rank":null},"run_fingerprint":"0ef58d0a814f5de4a059fb5196ffd7ae05b81387edca4c3c6aacc644e4d8f154","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (xhigh) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6911ff6a58ce17c666f07ba411ed9207def115d4b2a5b1865be23734f8d0fa28","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Terra (high) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-terra-high","upstream_model_label":"GPT-5.6 Terra (high)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"b9efb20044764f5f487493c6391194f196e33560de886bc3d466aafa645b193c","result":{"confidence_high":83.388,"confidence_low":65.799721,"sample_count":89,"score":75.655431,"score_display":"75.7","source_stated_rank":null},"run_fingerprint":"597c68836ce2a1c8f0b88a6490332c47aa36099ed147d1883e2c096c187e43a2","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (high) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"381269c2f2d84511f51cadc51c02251f323284123e068f520f661aeed14fc46c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Terra (medium) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-terra-medium","upstream_model_label":"GPT-5.6 Terra (medium)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"e6078bcca4388e8d60cc35cabaa75d9ab8c7dcdf6f19c71d0e80d3c229e1d1bb","result":{"confidence_high":80.513874,"confidence_low":62.211226,"sample_count":89,"score":72.284644,"score_display":"72.3","source_stated_rank":null},"run_fingerprint":"32d3a1547da43b4aff69ed067bf2335bb4878a9a618c0b890a0b2ca972b16f1b","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (medium) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a84b129dce4a16ecd11a231b86348abc7a9fcd1c2833e75414992c296f937791","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Terra (low) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-terra-low","upstream_model_label":"GPT-5.6 Terra (low)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"3a15ce41392fa9db0fcbc28fd26f14294abda047f355236def736e3172a7f973","result":{"confidence_high":71.886696,"confidence_low":52.168612,"sample_count":89,"score":62.546816,"score_display":"62.5","source_stated_rank":null},"run_fingerprint":"fad4f8afcf9bfbc253f77bcb08d6a58c31031ced61fee159d47431a5d606a906","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (low) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"531dfe5a37e9a17f7b8a2f499254e0b697ce9da1935ca4af847a9781d1088f45","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-terra-non-reasoning","upstream_model_label":"GPT-5.6 Terra (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"a4900eb58b1f9452bdea45126b8884417cc873dad03340d94615e829aefd3d05","result":{"confidence_high":66.020116,"confidence_low":45.82802,"sample_count":89,"score":56.179775,"score_display":"56.2","source_stated_rank":null},"run_fingerprint":"bb0acb634a58eb8f31e0736072947ec697890fd1f6f410c640ac6a5cf2a77b9d","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_d522137a2b9b55b4156d183d","split_or_window":"single-agent-baseline","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"b0e7be9867e2d976df845ef4b0ff5cef96d31bc63a167ae6ab8da7b1cb554906","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · Terminal-Bench 2.1"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"single-agent baseline; exact harness not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"terminal agent toolset"},"run_count":null,"scaffold":"single-agent harness not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"35ec4343c781d4675f2f64710c5000fe0977214013c6f6f70121275e1c4fe180","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.4,"score_display":"87.4","source_stated_rank":null},"run_fingerprint":"3759d3b4b786fb0b2a5ac7341749c326109a224daab8f27e12edc3c818f2ff28","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · Terminal-Bench 2.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c6950733217e3e994d1cd41d","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T21:39:34.661007+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T21:39:34.661007+00:00","status":"fresh"},"measurement_id":"c11ea58daf33dc934917dad6b9ed9c08626c6a2c192a071ed482a921a4a4d2ff","metric_details":{"accuracy_stderr":1.4,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:07:20.142219+00:00","total_cost_usd":2484.52,"total_tokens":7024607919},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"OpenAI","url":"https://openai.com"},"agent_url":"https://openai.com/codex/","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"9e11b624-6922-4ac8-a26f-c38146b957d9","model_release_date":"2026-07-09","model_url":"https://developers.openai.com/api/docs/models/gpt-5.6-terra","n_trials":370,"source_row_date":"2026-07-12"},"run_count":5,"scaffold":"Codex","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"a3674846c9dcf29b2407ae67a492b1d523022ddcfd99d682b436215d9b3be416","result":{"confidence_high":31.383136,"confidence_low":13.118005,"sample_count":74,"score":20.81,"score_display":"20.81","source_stated_rank":7},"run_fingerprint":"763202f791382f7b7ef5da41a61a23d7342d7f54ffa25ffabfe00a8c2e77283f","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c0475cd2c522dfa5840c4d57f2f778ff5e7f71b02315019220181f79bfa1d7b7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Max) · tauBanking"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra","upstream_model_label":"GPT-5.6 Terra (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"20fed09e29bd16c3439eb190d21a39c8cbe7384bc9bc015b336c3dcc69d252a7","result":{"confidence_high":50.15653,"confidence_low":31.002041,"sample_count":97,"score":40.206186,"score_display":"40.2","source_stated_rank":null},"run_fingerprint":"e261aa1657d343758dc89cbca2db3f1daf45aa1298893d4b3df1aab299376381","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Max) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"03a750aed464d68c5df55428b1a6f06b9ecf056e78d4b284fb323b006affbbed","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · tauBanking"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-xhigh","upstream_model_label":"GPT-5.6 Terra (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"330b085e0700dc8778a97db954539d7212071379ea7c214601110523bcc1f434","result":{"confidence_high":39.415612,"confidence_low":21.513212,"sample_count":97,"score":29.690722,"score_display":"29.7","source_stated_rank":null},"run_fingerprint":"3bb424af7fca7883aa233134a108be61324e690d89123aefb0fe3bfbf5353fdd","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"480f8b861459ac56cc96f8f205c5f209380fbb7674ae65eba3ce9158e895a400","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (High) · tauBanking"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-high","upstream_model_label":"GPT-5.6 Terra (High)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"881cf9545e6de3f87028459baf55a05a6219651c9e146db963a0dc6279ea823c","result":{"confidence_high":38.335634,"confidence_low":20.609881,"sample_count":97,"score":28.659794,"score_display":"28.7","source_stated_rank":null},"run_fingerprint":"b5d02adce3ed4c400df86cab71b3f16a5641d66f28ba35e00e6e266abe975c7f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (High) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e3d6284733847ff006dd0c9e4423cbf4e2b68484e08d25aeb1a92b4815d277c5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · tauBanking"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-medium","upstream_model_label":"GPT-5.6 Terra (Medium)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"5863115a530b7056f16b3cc468724a086d31b500566ff9b6f2dec12123db6207","result":{"confidence_high":35.063014,"confidence_low":17.932574,"sample_count":97,"score":25.56701,"score_display":"25.6","source_stated_rank":null},"run_fingerprint":"7d6370df449bc8ccf1b481448660c5089e94553bc20e62d05708cc5f8ef8a846","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"379001e8619b140897b752a37d56331b58938bb636a8d34ebdb4177f96c8a1c8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Low) · tauBanking"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-low","upstream_model_label":"GPT-5.6 Terra (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"678442090368ae212e948a2466d79f9e5191fb7ea382be04633732003792b17c","result":{"confidence_high":27.66538,"confidence_low":12.240374,"sample_count":97,"score":18.762887,"score_display":"18.8","source_stated_rank":null},"run_fingerprint":"9821f8947b09897f86c51bdd1a8a397d5a09eaaae07cc760a1c20841bfc5e4c8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Low) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"57880327e1bcfb7b9a06609df1e080ed4be9da7549dc9c503a2906ebe915edb0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · tauBanking"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-6-terra-non-reasoning","upstream_model_label":"GPT-5.6 Terra (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"d0d449bcfd12ebfd9f850ee144f67f3c3b0a00848c1fef803d57e97f7bb89569","result":{"confidence_high":24.192609,"confidence_low":9.763218,"sample_count":97,"score":15.670103,"score_display":"15.7","source_stated_rank":null},"run_fingerprint":"e205affa493a30a6f79f164a944ef4f8e93d0670ccf61076e5c852ec1495a1bc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aa5cbff51942e6a67e2cc8f518474990848309a152ae8daebc51b7fe789d8aa1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.718","mean_standard_error":"4.2","model_release_date":"2026-07-09","notes":null,"standard_error_points":530.0,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra","source_effort":null,"source_model_version":"gpt-5.6-terra_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"a1668feb2153a181937247e8a8238c74dcc7c224ec9eb258380fd70180ae2a3f","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":58.20000000000001,"score_display":"58.2","source_stated_rank":12},"run_fingerprint":"2f86d34b28c1d1e829ec6cf26df865967770a83940895d07812d384d5cb6c650","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"eba04e9a079eb8ade98550f25a697cb6917a9c601ef4de77945226104c2ac474","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":23457},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 5.6 Terra (xHigh)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fbeade1dcc928ed827479dcc71179483ef1eccbb7eb52563f7ab2f5e8e64fe6e","result":{"confidence_high":0.015615413236078318,"confidence_low":-0.00755816966171599,"sample_count":1686712,"score":0.0040286217871811645,"score_display":"0.0040","source_stated_rank":31},"run_fingerprint":"ef2fd31b1f77b8d02ab40db3d3da7a3d63092905b24633d172ecd4ca4b736239","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"90b02a4926c9982c399cff69dc0e07d5b2105f1137532646d13e5bc6a6668ff5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Terra","source_effort":null,"source_model_version":"gpt-5.6-terra_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"a513e3bc1170493bbe4a4f0ea74a94f7ada17a29a737518886e56958e0c294d1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":7343.206,"score_display":"7343.21","source_stated_rank":12},"run_fingerprint":"b859e66a3f49f2c4dba616dcf2fca65aa52789481dd90061ca772669e9aa18e1","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0770c98a055bcbf0b3bfe388","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2e3a10e8804c415d39192599b79d0a4a87389486fdcab2aef21981e711b4a48e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-18 / +18","source_locator":"Leaderboard models table · GPT-5.6 Terra (Max) · gdpval"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-terra","upstream_model_label":"GPT-5.6 Terra (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"60a7fd1f430813dfcebecfbea231211e5d3ead7fb8264799e41771cd695e591f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1453.21,"score_display":"1453","source_stated_rank":null},"run_fingerprint":"717fd707822e9f42ed5d64c064aa8de5a87be7cdb91ae8cfa3dc68cf043e64a3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_7a539a480ccf940eda2cf169","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f618155ad84e7b57653bc98c2beb43ad2ec994a655044ccca06a8309c591be49","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · gdpval"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-terra-xhigh","upstream_model_label":"GPT-5.6 Terra (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c82691d91da76edd639a34112c39ba6d52540dcb100428a6c90bf1a78a094a54","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1428.16,"score_display":"1428","source_stated_rank":null},"run_fingerprint":"671538b345a55ee85820ba4a7a840fafe20550c536ef3a8137b7d9bc759d0791","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_10b9eba1355d3375994a7c40","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"23ac903025989e795afaad4d783e9d013631533109b2b2dc10b04f44d2358d83","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · GPT-5.6 Terra (High) · gdpval"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-terra-high","upstream_model_label":"GPT-5.6 Terra (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"02a265543b1c4c551ab6b12cfb8a3eebb70ff003e1d44ed2728e3b03cbd22d78","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1362.55,"score_display":"1363","source_stated_rank":null},"run_fingerprint":"d078681b6ee08f16081d740c00bcf75e3fde820e7a766d8551bdfe88d2926277","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_70294a052eedaa815d9395a8","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"22fedd0dde79abe718ee731feaea9f4da9850763ded49dacad461ae02182154e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · gdpval"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-terra-medium","upstream_model_label":"GPT-5.6 Terra (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c16799a1fb0e9f507394765fcf9e7476b9917d32329257bb260725d423c86d4d","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1254.61,"score_display":"1255","source_stated_rank":null},"run_fingerprint":"500ff8dac942f05d9270b37e0c578e78a38f213c6d32e13189f0a4685e84a6fa","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d80da344491cf97973b6e76c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"edf49ef2810419820a8730340cdc3df1951cc7878d6d8cd78dde58e73b7956ac","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-5.6 Terra (Low) · gdpval"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-terra-low","upstream_model_label":"GPT-5.6 Terra (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"241404b29240561c849f16193a3b8b0951b227a5ce242ef3a1d608b802d87326","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1096.06,"score_display":"1096","source_stated_rank":null},"run_fingerprint":"2c9dc97b77bfbbc55ab7fb22d2f83def549e602adb0aef74c9c5b4aab498c621","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_9db5746a8adbf46d3197bec3","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5d856b46632443a3962cb02d0eabd9540cb1edaba8dab11993b62d05fc780d51","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · gdpval"},"model":{"id":"openai/gpt-5-6-terra","name":"GPT 5.6 Terra","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-terra-non-reasoning","upstream_model_label":"GPT-5.6 Terra (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"37d173e3b8e85c935982e91550e717847014bf4eb1684e6e3a507de220bdb7c9","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1084.77,"score_display":"1085","source_stated_rank":null},"run_fingerprint":"38ed80a35cb0ecb87b887197fb39fdc8833000a8459f7b722bc7feab660fcfc9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Terra (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c3c77fd072077fc1511b52ff","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"07d95739af05459c0828291060959b76a0c17071991cf454a51fbae9e05374e6","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-23"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.5; model release: 2026-04-23","source_accessibility":"API access","source_model_name":"GPT-5.5","source_model_release_date":"2026-04-23","source_model_versions":["gpt-5.5-pre-release_xhigh","gpt-5.5_high","gpt-5.5_low","gpt-5.5_none","gpt-5.5_unknown","gpt-5.5_xhigh"],"source_reasoning_efforts":["off","low","high","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"aeab45f2f0a6fecb3a648f8cc89499ffff39ac3e025e43379a9801d5861fe7ce","result":{"confidence_high":162.37,"confidence_low":156.87,"sample_count":null,"score":159.22,"score_display":"159.22","source_stated_rank":10},"run_fingerprint":"494049fab6311a19ecf2380312bef6988e243bee0f421d5a765e3d9d103711d7","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fc1802c65e18256a1d9ae18b","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bdbe84038d2859d73e61b9382df5474b20d0c3e6cf69c6482d1b1f5a7ebf9f2e","metric_details":{"license":"Proprietary","variance":3.6216467098495415},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1485.1662276199647,"confidence_low":1477.7063598734187,"sample_count":69202,"score":1481.4362937466917,"score_display":"1481","source_stated_rank":25},"run_fingerprint":"64af9dce9ca35de6ac8e2370a8f0632cfaddf2fbe30d5fdf689a33064408e7e2","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"93ec9e00a391cad2bbb741a75b77a2ff460b6e22a7ee2450b41ab5657af8cac9","metric_details":{"license":"Proprietary","variance":3.559126279636996},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1480.4426929815688,"confidence_low":1473.047495339619,"sample_count":70781,"score":1476.745094160594,"score_display":"1477","source_stated_rank":30},"run_fingerprint":"c5c39c83331baef362f5fb0bc0b3b5334bc73dc21c4ecaccec3fdc32087f0351","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ab8bb327eb304f72fdad66c9","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"297c16d4ec74ff085951eac8528376ea4d8b935cf025dc7d1e6197496a623d2b","metric_details":{"category_scores":{"Agentic Coding":53.989666666666665,"Coding":82.14949999999999,"Data Analysis":81.57566666666666,"IF":70.72925000000001,"Language":87.363,"Mathematics":95.85725,"Reasoning":89.65375}},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.18829761904762,"score_display":"80.19","source_stated_rank":11},"run_fingerprint":"4f9917dec0c2907b9d948e8d46e21fe54944a3dbe8a6db8ca5370d4710a625f0","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":165,"at":"2026-04-23","basis":"evaluation_at","evaluated_at":"2026-04-23","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"da050b7724b533a9c6096f7683fd23357b538b34e9c9028bfa60b7b55a8f4182","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/openai-gpt-5-5","source_model":"GPT-5.5"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5ff7843b5cc0b28f60ea81140d0fc83e0c7b0fb1123ff61449f0cf6367932ab9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.56,"score_display":"43.56","source_stated_rank":8},"run_fingerprint":"c4d00fa87778d3503e0da375a8106b8ec51aca1d5d26784025c8444e5f851cdf","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ad9dd60f09831d08eff03512","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:28:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:28:25.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"104d45dc79fcc992e22fb22b9316c6a18ec1cb6e96150b492dc4e94c9fca9fe3","metric_details":{"best_score_across_scorers":"0.63","model_release_date":"2026-04-23","stderr":1.5275252316519465},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"TijCafTTJUFrJ6jPGvBbgq","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTijCafTTJUFrJ6jPGvBbgq.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/TijCafTTJUFrJ6jPGvBbgq.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d23a2d7fa0743d0208611541cc6748e0c323c4e88079d09b9ec90ad86c184457","result":{"confidence_high":65.99394945403782,"confidence_low":60.00605054596218,"sample_count":1000,"score":63.0,"score_display":"63.0","source_stated_rank":13},"run_fingerprint":"be77395ea7f1ed7de7aa3102ffaffdc80b806991df576a00905189305e07d310","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cb64fd7b19004d8a341d10ae6d5e09c3d4dc5a46e9fee286bb7efc52f25f049c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.082767,"date_is_proxy":true,"latency_seconds":112.326,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.288},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":93.434,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"xhigh","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.5","zero_shot_accuracy":92.929},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"a9b0bdd00d093b3d4216aaec1d9ec0048d580efd845cba683fd9fdd7682269e5","result":{"confidence_high":95.70648,"confidence_low":90.65752,"sample_count":396,"score":93.182,"score_display":"93.18","source_stated_rank":10},"run_fingerprint":"4d708b4f1e53dc4cef8ff017352c21444e2cd26feb991cddab1896f7c84710b7","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":164,"at":"2026-04-24T13:35:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-24T13:35:12.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f7addc6f0005df7cb07ba18d5a5d1a0761de8f01daab7566f8abe20a15a7aff6","metric_details":{"best_score_across_scorers":"0.9400252525252525","model_release_date":"2026-04-23","stderr":1.546977117727751},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FUv4EUwR78nAsdGbiwiLEr","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFUv4EUwR78nAsdGbiwiLEr.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/FUv4EUwR78nAsdGbiwiLEr.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"3c708c9ff34e2de87423e303ab61ff5ba32b80443d04fb5c26e8ebe28d639c16","result":{"confidence_high":97.03460040327164,"confidence_low":90.97045010177885,"sample_count":null,"score":94.00252525252525,"score_display":"94.0","source_stated_rank":11},"run_fingerprint":"fa57f8aaff43df03e01063005452f7b08e6947027748d301d38a0f85324230db","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":152,"at":"2026-05-05T18:00:18.624Z","basis":"evaluation_started_at","evaluated_at":"2026-05-05T18:00:18.624Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1132b5e6526f3baf25b8b48ce2ad072cc756790019388dd6c747c3e76e924591","metric_details":{"best_score_across_scorers":"0.9065656565656566","model_release_date":"2026-04-23","stderr":1.7532658377721708},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"2CdomqtBVz2it7a2Zx2G2t","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2CdomqtBVz2it7a2Zx2G2t.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/2CdomqtBVz2it7a2Zx2G2t.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d4ee8450b39b3ad72aaf7d2d2a148031138d20239ee0d4ac5fa0bc88da0eb0ac","result":{"confidence_high":94.09296669859911,"confidence_low":87.22016461453221,"sample_count":null,"score":90.65656565656566,"score_display":"90.7","source_stated_rank":39},"run_fingerprint":"fd6a373e4f55f27589f43795bce6ab3e6770921116e99939ad2e93bab9c9fd12","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:57:11.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:57:11.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d5bf0259a012020322b9b2e1ec69d5ef148df946dfbd0a36c0cc814bea46b998","metric_details":{"best_score_across_scorers":"0.7727272727272727","model_release_date":"2026-04-23","stderr":2.9857515673386437},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"KesKa5EXtFG7H4UMBFzgCh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKesKa5EXtFG7H4UMBFzgCh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/KesKa5EXtFG7H4UMBFzgCh.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e69c88d67179f3264111221c251df8aeb18f92c6528f826d61ce5e831465b53b","result":{"confidence_high":83.124800344711,"confidence_low":71.42065420074353,"sample_count":null,"score":77.27272727272727,"score_display":"77.3","source_stated_rank":141},"run_fingerprint":"b51be3eeac82d0fa3fdbdbd38b46b8a56e1c9e9adadab397e76f751fd7679a6c","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_528b7e4d648d2d13b940d84f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f2f8eb675d5a4d14d203e84f","split_or_window":"provider-reported-protocol","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":165,"at":"2026-04-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"c45bb80b9298f8d30ac9db9e00edd73ef2ecc2329813f51a4734396e6fe68aad","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Academic · GPQA Diamond"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort and tools not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-23","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"1a0f1b43eef5cfe12ea649ac550dd6e8d9837fea239fa2533813431a6536bbeb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.6,"score_display":"93.6","source_stated_rank":null},"run_fingerprint":"53035b417a916260b491a0caa89de3ada0b2c982c20b00b7763cdae5495bbf3c","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Academic · GPQA Diamond","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0ff972701cf87de48b97996","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"abf515f265fd4773bf9848356efbc1ca61977540256a62c1e5a60e6f06203dcc","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.027891,"date_is_proxy":true,"latency_seconds":42.114,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.318},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f15d35f138425a11f5202adb39f23438b44d82a67cf86db485e1e9b5fa466337","result":{"confidence_high":88.76728,"confidence_low":87.52072000000001,"sample_count":null,"score":88.144,"score_display":"88.1","source_stated_rank":21},"run_fingerprint":"12a87f009711bb2defec2be1ad77da1f6ee3a00187cd1e24706fcb804a778641","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":164,"at":"2026-04-24T14:06:46.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-24T14:06:46.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"916bfc1504db2ebf3e638c7c4f1d42d6ed475748fc1cd7641e156ce2964451e3","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-04-23","stderr":0.0},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dP6L7ViZSdcTiWxooLCTUh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdP6L7ViZSdcTiWxooLCTUh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/dP6L7ViZSdcTiWxooLCTUh.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f9d40076f0612e117e2b174313bd5a5f12bc67638455177ea0b9c021cc0b25ca","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"ed7864bfe208b2da620fe8add8adfdcd2cd93d47ede26f84846d10f447131515","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:39:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:39:05.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"52c7702f8d3cd3b612b33d89210bba804c0aa0f00658c70eba4ae8816801b939","metric_details":{"best_score_across_scorers":"0.8444444444444444","model_release_date":"2026-04-23","stderr":5.463890236888291},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CdmtwZ8NR8YeTXWiLVTRDJ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCdmtwZ8NR8YeTXWiLVTRDJ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/CdmtwZ8NR8YeTXWiLVTRDJ.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"2874e63b2ec74992d348280ae234c3d1782867cb7ea1941f02642bc7bb0e7c5c","result":{"confidence_high":95.15366930874549,"confidence_low":73.7352195801434,"sample_count":45,"score":84.44444444444444,"score_display":"84.4","source_stated_rank":93},"run_fingerprint":"73461df5142b3d27b9de8be2702c019d77bfab7e6d003e3a8946f9a02a54cf9e","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:53:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:53:58.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c98daac87bc440ce4d8d72f3fadb6508f7f308284af97bf999df6bfa4b2dccef","metric_details":{"best_score_across_scorers":"0.5777777777777777","model_release_date":"2026-04-23","stderr":7.446027270295804},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"QuqPmMUTbiEsUvoFYZx5Mk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQuqPmMUTbiEsUvoFYZx5Mk.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/QuqPmMUTbiEsUvoFYZx5Mk.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"59ba5d2e68bf8b14b13c542fbc45c5822023d027ee845d4fd63b5dcd6b8d527a","result":{"confidence_high":72.37199122755754,"confidence_low":43.183564327998,"sample_count":45,"score":57.77777777777777,"score_display":"57.8","source_stated_rank":169},"run_fingerprint":"5c41dc4960532f288593852f834016b6febb9d4b5ca8ea8fd9911c6ec313ff6c","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_528b7e4d648d2d13b940d84f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T17:47:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T17:47:36.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"658e06651f4b2dd3e444148e4f97dd2b202f5c15d8aa004b2f9565abe40ba478","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.8526315789473684","leaderboard_private_problem_count":285,"model_release_date":"2026-04-23","public_example_count":10,"stderr":2.1034091168852487},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"5FGXKvXQwGPtcCX2PNrLR5","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.5 (xhigh)","thinking":null,"upstream_model_id":"gpt-5.5_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"8cf0f33d75899c3ad5c42caaddf510069e92d52e9361c57eb73579d380f1126f","result":{"confidence_high":89.38583976383192,"confidence_low":81.14047602564175,"sample_count":285,"score":85.26315789473684,"score_display":"85.3","source_stated_rank":12},"run_fingerprint":"f961403560900ce98edf06ae83564970d45b147b00bee4be9fa2f0c5decf8e3d","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3667d02a092777603073cc2009ba6a4b0fcc09f6801cb15380b50e755f40a631","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.87,"model_release_date":"2026-04-22T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-5-2026-04-22-thinking","model_type":"CoT","upstream_model_id":"gpt-5-5-2026-04-22-thinking-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a0cfcd82b3d8405c11033eeb7ea6d86ff25e74d6a78b9934a39c06f28c60095d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.0,"score_display":"85","source_stated_rank":29},"run_fingerprint":"7c968f74b4cfb4536afdf548a752a3395f788f8aaea715876f4669b280defbc6","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ace205821b4c5b657db5d0b8d75e5958faa46173f3b0436bd8eee91d542f6d3f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.45,"model_release_date":"2026-04-22T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-5-2026-04-22-thinking","model_type":"CoT","upstream_model_id":"gpt-5-5-2026-04-22-thinking-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c6e3b6f77466a625fc97b5df06432ddf987ed546ab9d0ccf9ae1f28e16603b1f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.33,"score_display":"83","source_stated_rank":35},"run_fingerprint":"c44e0baee5780d9542fb80fbd8de6d19e0250d136327c48b7e26229697a76198","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3f182eb4bbfd0185d43ef10b421034039f3d0d2ddfb7e66ee038ed59ef8d0c9d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.86,"model_release_date":"2026-04-22T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-5-2026-04-22-thinking","model_type":"CoT","upstream_model_id":"gpt-5-5-2026-04-22-thinking-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7641d80d875f43867949bb745f34103160b1604a122135b7010d9fd8ca1b718a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.42,"score_display":"70","source_stated_rank":53},"run_fingerprint":"be799e20fed68d576cb3a67bf8960baae48c95742bd02c7a1a1625067e21519a","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"10618c0247519e92052b41712818fb06339b1971635859583fa8fe1866815b28","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.35,"model_release_date":"2026-04-22T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-5-2026-04-22-thinking","model_type":"CoT","upstream_model_id":"gpt-5-5-2026-04-22-thinking-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b2e0815d80137c6073b7c307ade9b58d02f5fb27fc30f774424ce9dbd9dab650","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.33,"score_display":"33","source_stated_rank":112},"run_fingerprint":"11ff6a63ccd88eb438903a417a56aae9ecbc39b046270a0ada045c3eda1e47fe","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1053c553c3b056c6023acb7cbfbddb7a7edf89ab0c98689efb4a11e8f966958c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.87","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (xHigh)","source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"04cf0fdb4a51c666f2ce5c4c2de1151829feadf2220c31f49f48a500720a0018","result":{"confidence_high":88.318079,"confidence_low":80.94283,"sample_count":360,"score":85.0,"score_display":"85.0","source_stated_rank":29},"run_fingerprint":"b924e6111cbbb8bf50c312d6d8d874fef09dc7c03857f3eea79b68600a6d0302","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"27fe2577bb5c71c4d9c1c4ca6ff9c863e4626e384479ad29b9336e2e6bccf61f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (XHigh)","source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"261dd8b20852c12666ba21c889d3a09af627f453c3694ccaa9aa02b81a033400","result":{"confidence_high":88.318079,"confidence_low":80.94283,"sample_count":360,"score":85.0,"score_display":"85.0","source_stated_rank":29},"run_fingerprint":"eacc8f9b5eb9f09887aab08bbee5aff136c1821651f88242bfb2b163e2232765","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"19fa399f15d6e2054c5bb3f00fafbb11188e0431d1d6b26883d62a6cf5a77c8c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.45","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (High)","source_effort":null,"source_model_version":"gpt-5.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"4de5aa0d7444c0a783c2ec94022f7cb8f75a95a67801316f78f6f8dde1dc5505","result":{"confidence_high":86.823954,"confidence_low":79.132221,"sample_count":360,"score":83.33,"score_display":"83.3","source_stated_rank":37},"run_fingerprint":"78075d1931b365d8c360553b152ffa8e338939bc63c29f76fc46dda2e03c6b6b","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dae0f57b7b696330e37efb583a147567cf5d1c44ed16fe2bb1a840e549e2a8de","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.86","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (Medium)","source_effort":null,"source_model_version":"gpt-5.5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"15fb0c4a3c5e04a3a3d351c9f4077b7f8ab909667cb05a7d7456f2f5e52a0461","result":{"confidence_high":74.899071,"confidence_low":65.509722,"sample_count":360,"score":70.42,"score_display":"70.4","source_stated_rank":56},"run_fingerprint":"b391a2d2c82546cbe9482cca15e36e6a1baa2aea84291758ad21403b180af1be","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bc7dcc5dbee71ddf1303a6d45b59633f39592c4ee46fe783fdef30d18bfb38c8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.35","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (Low)","source_effort":null,"source_model_version":"gpt-5.5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"98e1be1d342bf137e8b0dce7e26ec28c2312835acdb6e1dd0bf6d8563581f628","result":{"confidence_high":38.352968,"confidence_low":28.659051,"sample_count":360,"score":33.33,"score_display":"33.3","source_stated_rank":112},"run_fingerprint":"940b307c7e36525dd7d20c33d9ff7008359cd8111ccebdd297d79c37fc322d81","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T17:47:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T17:47:36.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c053b9fdd7cae592c189e962c30c53ffe186a3eed531917d708a780a838987b7","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.725","leaderboard_private_problem_count":41,"model_release_date":"2026-04-23","public_example_count":2,"stderr":7.149950690165275},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"PKSMspJiJdNThB6qNvaBSM","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.5 (xhigh)","thinking":null,"upstream_model_id":"gpt-5.5_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"c6fe6d4b6009dd1d72f17d21a085b69ed2db4cd517d5f245cf16da2b7779c620","result":{"confidence_high":86.51390335272394,"confidence_low":58.486096647276064,"sample_count":41,"score":72.5,"score_display":"72.5","source_stated_rank":16},"run_fingerprint":"0e35f3c6f92ca97ccb7657873433d018cc695f6f6308c5d6e16edc0bd854aec7","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8a9adb7875c5fd0fcce5e3ac47bf6ced3f83f363cea3810dbeceaa7dd6914c68","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.043319,"date_is_proxy":true,"latency_seconds":54.147,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.774},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c36041418f2f2eb437fe44df2ef761c854928b53b8cda9f520ef7f198276a90a","result":{"confidence_high":89.78304,"confidence_low":86.74896000000001,"sample_count":null,"score":88.266,"score_display":"88.3","source_stated_rank":8},"run_fingerprint":"7044bd1823246cb9fb85a47d82291a7c2edfe08a1bb7a5f598078829d5a75022","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_39a5bff9a48a29bea5ce760d","split_or_window":"standard-no-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":165,"at":"2026-04-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"7159d9f0a2edc2984675a19815432ada630b7c88f2bb1d8d99d5367f9e62520d","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Multimodal · MMMU Pro"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-23","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"230f77b744fff1a7ba46f873aafd71eba3c67c7bf8d603b635e31ef16d8039f0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.2,"score_display":"81.2","source_stated_rank":null},"run_fingerprint":"5d5b9e91b3711cbccd23f21573d46f0c90188e5b6dfeb820ee04d8e257121710","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Multimodal · MMMU Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0ff972701cf87de48b97996","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_aa0096763ba73cd85e46674d","split_or_window":"standard-with-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":165,"at":"2026-04-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"7833af27c01cd0bfa9610849f980b949485d97702fa6d1877181b25910b8a369","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Multimodal · MMMU Pro"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-23","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"tools enabled"},"run_count":null,"scaffold":null,"tools_allowed":"tools enabled"},"protocol_fingerprint":"05218c7cfa6407103c43def05de9a77e65a6475907a0ed9e4c25f1c4fb13cfd4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.2,"score_display":"83.2","source_stated_rank":null},"run_fingerprint":"e2097de7a72263d16303f4024f9276ddda3dcfd22aaa7c44b628ac42724e9af3","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Multimodal · MMMU Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0ff972701cf87de48b97996","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"79ae979034b781b7a9ffc909a61f6efb0ad148b3d73bc3d0a2a04bbdb860e232","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (xhigh)","source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"cd14415396135ccc6ba5ee353c723721ba102d2fce37c6cca75f60816df984cd","result":{"confidence_high":38.459269,"confidence_low":18.172948,"sample_count":71,"score":27.142857142857103,"score_display":"27.1","source_stated_rank":31},"run_fingerprint":"a04522143545476da4a1682b45c15af5342ad8013d07fc0da5e04a424dc2966a","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0c093ca1bf55071358129d5e80ef4aa5b28c50a71ff44aa06ebcaf86aec632f8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (high)","source_effort":null,"source_model_version":"gpt-5.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"0ea26da9e9afa07ea80f7bdb06cf2623ae90643d8a7b25c0e3209d73ae912bdc","result":{"confidence_high":36.635896,"confidence_low":16.743737,"sample_count":71,"score":25.428571428571402,"score_display":"25.4","source_stated_rank":37},"run_fingerprint":"8a7f04d4ab9bb635600ec7f61326ce48743d091a9c3aaa95167bc0a9b7a49887","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"73d802c37758d47e257d1445179f4149023e8efa843179088a74b697076f888e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (medium)","source_effort":null,"source_model_version":"gpt-5.5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7009b9672a6e3ee2cda861a0e43235a115925fb2fa4818f03d783f4eeb82096e","result":{"confidence_high":29.141525,"confidence_low":11.227773,"sample_count":71,"score":18.5714285714286,"score_display":"18.6","source_stated_rank":58},"run_fingerprint":"c7f4d8acf49697052d2e38cb2cb20c58ac97c2d886c210afd035d3e8dc01e8a4","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"814561da8ab8f3b2b46f5fb28eda5126cbece8e3a6e16444db0affd9611cef49","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (low)","source_effort":null,"source_model_version":"gpt-5.5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a0c8afd7a3569a5dd05a5f09d318459670c47ed040781d1f6b7270a0973f29e0","result":{"confidence_high":16.669402,"confidence_low":3.642296,"sample_count":71,"score":8.0,"score_display":"8.0","source_stated_rank":101},"run_fingerprint":"cf2fd53f7bbb145380e49da390f9ff3698665d8b1ecda91ff858401a86bb8f34","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"148f81b0df4520469f116c20fe5cf46c78b201664862edb0c939e1ea31179e14","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (Non-reasoning)","source_effort":null,"source_model_version":"gpt-5.5_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"d2815933a7c439a0cdb088df7a4d1452c6cd02b1624000fc7778ae2a134ecef7","result":{"confidence_high":7.588324,"confidence_low":0.255137,"sample_count":71,"score":1.42857142857143,"score_display":"1.4","source_stated_rank":136},"run_fingerprint":"9f9721e556471b4bcf03d8f0b9ab13ebe7fdaf6fcc88298347b7c9f66015aea2","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_528b7e4d648d2d13b940d84f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"33981b8e3eed40fe92c8c2e8620842403f58554627a4d8fe894771a0a107198a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5_unknown","source_row_date":null,"upstream_leaderboard_url":"https://lmcouncil.ai/benchmarks"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"24ae4b1c43dd9aa74ec0e566041f1e4cd39e0181e5c0b9bf014e1d881e750941","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.0,"score_display":"69.0","source_stated_rank":15},"run_fingerprint":"da04c6c6ebdbf38f2881c3c8537e18aff6f51f3d5447d0ce5ea9432582dbb5df","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cfd072d0392af1ffbd5bf34e","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":164,"at":"2026-04-24T14:20:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-24T14:20:22.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"27d028c68c82822a56b6ba86134a0f8b64b9ba62e0d213b9628c6e03012a4861","metric_details":{"best_score_across_scorers":"0.54","model_release_date":"2026-04-23","stderr":5.009082659620331},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bCy8WkKexRgzZoX2oJbQNM","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbCy8WkKexRgzZoX2oJbQNM.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bCy8WkKexRgzZoX2oJbQNM.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"61a1445a574cc06fcaab9daabbb98d9dd01e28788ca5d714739567119d436a69","result":{"confidence_high":63.81780201285585,"confidence_low":44.18219798714415,"sample_count":100,"score":54.0,"score_display":"54.0","source_stated_rank":9},"run_fingerprint":"ffcecf58a44715443dd5c57d07fdcfd0c3334e72812f49769d7831becdf67986","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:50:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:50:05.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"58fa25fb7957cd58d3caae30d3f9c7ba5a0c22dcdc1a38379342c29e8ef15503","metric_details":{"best_score_across_scorers":"0.26","model_release_date":"2026-04-23","stderr":4.4084400227680804},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"EnbN27oUrNyfbxuPwtZey2","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEnbN27oUrNyfbxuPwtZey2.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/EnbN27oUrNyfbxuPwtZey2.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"54ea70520a908d7a3f62225e56cba9cca75ddeeae8c4a4a63d45be6991c7bd94","result":{"confidence_high":34.64054244462544,"confidence_low":17.35945755537456,"sample_count":100,"score":26.0,"score_display":"26.0","source_stated_rank":62},"run_fingerprint":"3abe8a8d0067b75e52ac7cc47374c36765b43fe833cf484464c58cc0f312adb5","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T01:00:18.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T01:00:18.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0d5395c4dfccf6c3dd30eae200ade4d107ca90906eac939d30f9df4c77146096","metric_details":{"best_score_across_scorers":"0.1","model_release_date":"2026-04-23","stderr":3.015113445777636},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"XRVSYnR4bhQ8Zb62Un7GrS","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXRVSYnR4bhQ8Zb62Un7GrS.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/XRVSYnR4bhQ8Zb62Un7GrS.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a029e9ac950eae12674e81e0ef54f9c6f89e9ea77bceabeb69c73a758f7b1be1","result":{"confidence_high":15.909622353724167,"confidence_low":4.090377646275833,"sample_count":100,"score":10.0,"score_display":"10.0","source_stated_rank":137},"run_fingerprint":"571e4aa4e7f261bec8124031840694754427c0d936ba130b0dbb1ae308b2deda","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_528b7e4d648d2d13b940d84f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"eae216c164589a941117927d805fd71caf92454efb7e7bd4b09c9a8134ee3a82","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.73","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (xHigh)","source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b27c8f16f8fbb9f7481f43ff962202ec8ac65abac8edd3ddaa974eacb7d78b33","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.0,"score_display":"95.0","source_stated_rank":33},"run_fingerprint":"4e0a288c63170d067d0df4813f1822d6736f0e56b0b493b927fdcb3e3cc0d2a5","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d0f9075ed33916deb64906aa4036a4ee672be886ab01fe2acf801720161869b3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (XHigh)","source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6b4650f5475a9c0996416d8c3a6e6b6c2b22099ff6f02d609c60731c101a629d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.0,"score_display":"95.0","source_stated_rank":33},"run_fingerprint":"58cbef3c9df70c881971aadcdcf8c00004cba3cb389ff66a2a77438a885f0f1a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bd70fdef459b26903638793b3f759db0d461b7dc336ca59d829590d3b1a79cc5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.56","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (High)","source_effort":null,"source_model_version":"gpt-5.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"4021486679be0fd694c87d433fd54527f49f1ae03d4f1ce7c895101512af17f0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":94.5,"score_display":"94.5","source_stated_rank":37},"run_fingerprint":"c1b880e3055c20ed8b642d159f7cb183a945a4c2d3e6fbf2f300210617f91780","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0a7f9686803b3fa65bf53f0c3c7219265c011fb594bb16ac3f8ed38a15391e6f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.39","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (Medium)","source_effort":null,"source_model_version":"gpt-5.5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"26de432fffe16b73866f2adeca443fce7197d2e698aae89e1abc305bf286e0fc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.17,"score_display":"92.2","source_stated_rank":58},"run_fingerprint":"c0dacb66698bb2105ea7b7c799a12c8d50a63c6a1afb539953bbb29bcc064c14","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"425a81c65afc0823b3234a956fab4ccff7990f880eaa372d778ecae5468b33f8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.2","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (Low)","source_effort":null,"source_model_version":"gpt-5.5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"987bd68d1fdad35b5ecac11eff190b33154e9c6c953e3452821ab907a9fc26c6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.17,"score_display":"76.2","source_stated_rank":121},"run_fingerprint":"6edebac86df7d5c05aa821ce6f343b727cc4db11950538d7ef933dc2e0b90d65","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":165,"at":"2026-04-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-23","status":"stale"},"measurement_id":"b5c5b2f619a3f983db8a9f6743be4e0ec84be9da9a24eaa56beca956b003375d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run; vendor scores using a different agent or tool setup remain separate supplemental runs.","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","cost_per_test_usd":1.361765,"date_is_proxy":false,"latency_seconds":426.425,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/openai_gpt-5.5","source_locator":"Results · Overall accuracy","stderr":1.695},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3f5f49706c56ea388326a770eeb10ab04810fc5e0cbea1239400999660bccb57","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.5"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"aba6944e319eab5f62a169f69bc5d44cd6e263406430225aaf46fb238da7f12c","result":{"confidence_high":85.92219999999999,"confidence_low":79.2778,"sample_count":500,"score":82.6,"score_display":"82.6","source_stated_rank":19},"run_fingerprint":"2751376fe747045640c60d186958e61d1f321a76df75d5ec09f33cc8b4a30fca","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":164,"at":"2026-04-24T13:52:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-24T13:52:36.000Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"455f2550e70956511279ab4dbe6fb65f2b72357403123989a31dd5d2d3e5789d","metric_details":{"best_score_across_scorers":"0.8057851239669421","model_release_date":"2026-04-23","stderr":1.8000198869416815},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LbhPRqUkXrs5KJ7vyT4CR7","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLbhPRqUkXrs5KJ7vyT4CR7.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/LbhPRqUkXrs5KJ7vyT4CR7.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"0ec4b199f738a10bf558abc49304700ce5f7a381e97461cae3b6f5202fd4bcf5","result":{"confidence_high":84.1065513750999,"confidence_low":77.05047341828852,"sample_count":484,"score":80.57851239669421,"score_display":"80.6","source_stated_rank":2},"run_fingerprint":"51b579d114b0428cf24c6196bf2bde9e35b6498bb7b87cf08da9d6f30f92e9fa","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_332cbf0e979daa9e173de9f4","split_or_window":"2026-05-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-29T23:53:02Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1229e757891d9bd44cf119f0014fe026cbe5a5f9f7263f5322e1db34b5a1f022","metric_details":{"comparison_warning":"Official post-release SWE-rebench window; use only for same-window comparisons and never mix it into the canonical rolling-window rank.","comparison_warning_code":"swe-rebench-post-release-window","model_release_date":"2026-04-23","pass_at_5":71.42857142857143,"potential_contamination":false,"sem":1.5058465048420853},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-post-release-window","comparison_warning_code":"swe-rebench-post-release-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","supplements_range_key":"1772323200000:1778803200000","upstream_model_id":"gpt-5.5-2026-04-23-xhigh__tools","window_from":"2026-05-01","window_status":"supplemental-post-release","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"3d78907a601d1c50e69137936fb47907e786d140299fb0d62926c34f04f6ad6c","result":{"confidence_high":69.61812581615715,"confidence_low":63.71520751717617,"sample_count":null,"score":66.66666666666666,"score_display":"66.67","source_stated_rank":null},"run_fingerprint":"32b742003ccfcd025e4c3cff0917135cdef008a8f6af4242e8c943873503ae26","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-post-release-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_332cbf0e979daa9e173de9f4","split_or_window":"2026-05-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-29T23:53:02Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"870664cfd80c3dd02d5b5fd58496908e087cfb6b593009b4f5c9b343f88c507f","metric_details":{"comparison_warning":"Official post-release SWE-rebench window; use only for same-window comparisons and never mix it into the canonical rolling-window rank.","comparison_warning_code":"swe-rebench-post-release-window","model_release_date":"2026-04-23","pass_at_5":66.66666666666666,"potential_contamination":false,"sem":2.428104530282278},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-post-release-window","comparison_warning_code":"swe-rebench-post-release-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","supplements_range_key":"1772323200000:1778803200000","upstream_model_id":"gpt-5.5-2026-04-23-medium__tools","window_from":"2026-05-01","window_status":"supplemental-post-release","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"27bb3e055b14917fd66ed2d340622ba079cc3887a8748490ac8b30a439f8649d","result":{"confidence_high":63.80670392697232,"confidence_low":54.288534168265784,"sample_count":null,"score":59.04761904761905,"score_display":"59.05","source_stated_rank":null},"run_fingerprint":"6009f094fefca850d8ec1c08f4c36a63d6b6212fb035d24f57a823a2b3580906","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-post-release-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"fbe6ef96a5ee8be066f55f230b45f6fbb08722983c6db8b30c64e0fe72e2dba0","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-04-23","pass_at_5":70,"potential_contamination":true,"sem":0.9090909090909094},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","potential_contamination"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"gpt-5.5-2026-04-23-xhigh__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"5267691d46803160619bd91fb115fd7ea2abc4cbd70ba9f1ea118dfb15775d23","result":{"confidence_high":64.5090909090909,"confidence_low":60.94545454545454,"sample_count":null,"score":62.72727272727272,"score_display":"62.73","source_stated_rank":null},"run_fingerprint":"6430dd29962d77bb8d3a627bccc3e7734d33d0599830debfc533220453bf8375","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"b4a3891f7b2460b8369149b9648186c04e36367c8d416af48f1cefddb95277c7","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-04-23","pass_at_5":70,"potential_contamination":true,"sem":0.7820295697311489},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","potential_contamination"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"gpt-5.5-2026-04-23-medium__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"3018754e366c677d6385e26b7bff8944de690b2533f9967a89bfb18925665f0c","result":{"confidence_high":60.441868865763965,"confidence_low":57.37631295241786,"sample_count":null,"score":58.909090909090914,"score_display":"58.91","source_stated_rank":null},"run_fingerprint":"93bb162910605e8b51950f4e208c92e2fe3105f1039dd8f39ee40a9b5aaa8944","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-live","metric":"resolved_rate","name":"SWE-bench-Live · Lite","release_id":"release_9d2de9d5be7e9822af19f7b3","split_or_window":"lite-300-python","unit":"percent","version":"SWE-bench-Live Lite"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-03","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:10Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-03","status":"fresh"},"measurement_id":"4e6da051efdf6633f8df8ed2abea12affa4e49faf5fbd86ffed5460a67bb0277","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 300 scored items with a 95 % Wilson interval.","leaderboard_script_url":"https://swe-bench-live.github.io/leaderboard.js","publication_status":"cleared-with-attribution","resolved_instances":186,"submission_url":"https://github.com/SWE-bench-Live/submission/blob/main/submissions/lite/agav/gpt-5.5"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"agav-agent","report_file":"reports-0605.jsonl","report_set":"lite","source_model_label":"GPT-5.5 (Medium)"},"run_count":null,"scaffold":"agav-agent","tools_allowed":"agent-specific coding tools in the official SWE-bench-Live harness"},"protocol_fingerprint":"42e01a189064d8fb76dba75d73198590fe5d06830446eda18b634417e755a175","result":{"confidence_high":67.308218,"confidence_low":56.388339,"sample_count":300,"score":62.0,"score_display":"62.0","source_stated_rank":null},"run_fingerprint":"1ea059c82f68d5b634480b91c4aec9ac4a8b1ca83b2eff3476b335471f6964cf","source":{"content_hash":"e0226b6333c547885c15c6d53074d0ad5e2a7b99db2d6b54640d7d6537c242b4","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-09-30T18:53:18Z","homepage_url":"https://swe-bench-live.github.io/","id":"swe-bench-live","last_fetched_at":"2026-10-05T12:33:28Z","license":"Benchmark and harness MIT; aggregate leaderboard results redistributable with attribution (maintainers' confirmation 2026-09-02)","locator":null,"name":"SWE-bench-Live · Lite","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-bench-live.github.io/reports-0605.jsonl"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"32108dd002643a3c9c30777a8eb915c0a26e03f5f99d42aa1d69ff8d7d2ee499","metric_details":{"ci_attempted":452,"ci_half_points":6.465646340908863,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":303,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":82.01548672566372,"mean_cache_tokens":8032415.716814159,"mean_cost_usd":7.226236674778761,"mean_duration_seconds":1806.3251869889382,"mean_input_tokens":8372927.630530974,"mean_output_tokens":46294.72345132743,"median_agent_steps":76.5,"median_cost_usd":6.110482499999999,"median_duration_seconds":1588.3887945,"median_input_tokens":6648149.5,"median_output_tokens":44492.5,"median_output_tokens_to_pass":44036.0,"median_peak_context_tokens":141957.5,"n_attempted":452,"n_passed":303,"n_tasks_attempted":113,"n_tasks_passed_any":100,"pass_at_4_points":88.49557522123894,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_5_xhigh","source_model_version":"gpt-5-5","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"e069c3df0375db8dacb24d43ec010322abc67897fa4350149d14ef096dc0b9fb","result":{"confidence_high":73.50104457099735,"confidence_low":60.569751889179635,"sample_count":452,"score":67.03539823008849,"score_display":"67.0","source_stated_rank":22},"run_fingerprint":"68bdb8de629175dcfb2b193eacf397c5377ec172d2bd6730a1ae37f8c3e9fd6f","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"5af51356ed6f004cbbba2caf8c64a17adb002dd5e69d7ab90d0efb484c273ed1","metric_details":{"ci_attempted":452,"ci_half_points":3.1168426495054677,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":291,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":61.924778761061944,"mean_cache_tokens":4205870.442477876,"mean_cost_usd":5.100437004424779,"mean_duration_seconds":2048.0589126172567,"mean_input_tokens":4617781.876106195,"mean_output_tokens":31159.49778761062,"median_agent_steps":60.0,"median_cost_usd":4.582879500000001,"median_duration_seconds":1881.0672625,"median_input_tokens":3860162.0,"median_output_tokens":30377.5,"median_output_tokens_to_pass":30122.0,"median_peak_context_tokens":107945.5,"n_attempted":452,"n_passed":291,"n_tasks_attempted":113,"n_tasks_passed_any":102,"pass_at_4_points":90.2654867256637,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_5_high","source_model_version":"gpt-5-5","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"bd4df460923fcb51f11b2c736a62b9b22da770042bd8904a148d7d1e353e8a02","result":{"confidence_high":67.49737362295679,"confidence_low":61.26368832394585,"sample_count":452,"score":64.38053097345133,"score_display":"64.4","source_stated_rank":29},"run_fingerprint":"bd8c4cc88277cfc3ed66c6d42e94adc98c6d15c49ba1c219fa238d212cd68307","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"0d970878a5f8140d5d82cae7d5c424122acfc2b25357a8525fac4a5776f6a519","metric_details":{"ci_attempted":452,"ci_half_points":2.553087495290979,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":244,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":45.982300884955755,"mean_cache_tokens":2229359.0088495575,"mean_cost_usd":2.749342734513274,"mean_duration_seconds":1288.995063800885,"mean_input_tokens":2438539.053097345,"mean_output_tokens":19625.433628318584,"median_agent_steps":43.0,"median_cost_usd":2.436942,"median_duration_seconds":1132.512228,"median_input_tokens":1983652.5,"median_output_tokens":18737.0,"median_output_tokens_to_pass":18448.5,"median_peak_context_tokens":75466.0,"n_attempted":452,"n_passed":244,"n_tasks_attempted":113,"n_tasks_passed_any":88,"pass_at_4_points":77.87610619469027,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_5_medium","source_model_version":"gpt-5-5","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"622ea53cdb8faf6b7d654983af2bde800f6494ea89fedc7e8544af0f14b52595","result":{"confidence_high":56.53538838024673,"confidence_low":51.42921338966476,"sample_count":452,"score":53.98230088495575,"score_display":"54.0","source_stated_rank":41},"run_fingerprint":"85ba7054e3b5362c9696a0bf55ad9ed8e5427fafc950ae4acb405e6cb0fcd377","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"667cf7e70b11682b5461153fd06ad3bb960d1d8009280b2e16437c47ac6603be","metric_details":{"ci_attempted":452,"ci_half_points":2.294503222007181,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":122,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":28.06637168141593,"mean_cache_tokens":658930.407079646,"mean_cost_usd":1.2002025442477877,"mean_duration_seconds":569.9958436615044,"mean_input_tokens":776751.5840707965,"mean_output_tokens":9442.774336283186,"median_agent_steps":27.0,"median_cost_usd":1.108106,"median_duration_seconds":485.6083185,"median_input_tokens":645287.5,"median_output_tokens":9020.0,"median_output_tokens_to_pass":8910.5,"median_peak_context_tokens":39278.0,"n_attempted":452,"n_passed":122,"n_tasks_attempted":113,"n_tasks_passed_any":54,"pass_at_4_points":47.78761061946903,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_5_low","source_model_version":"gpt-5-5","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"2a71391d2c0f5da8466196e5a5b9b01ee8ba68885d88b4338e9ad63e43ff795f","result":{"confidence_high":29.285653664485057,"confidence_low":24.696647220470695,"sample_count":452,"score":26.991150442477874,"score_display":"27.0","source_stated_rank":66},"run_fingerprint":"d69985ee875849117bfe577bd96e428f70284a45c2129b077d6bf86cfc5a00fc","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0ba373ae03d5d9e918710ab09d31de138eaf2f5ea157e3c734e7502bd09b4709","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"82.01548672566372","mean_cost_usd":"7.226236674778761","mean_output_tokens":"46294.72345132743","model_release_date":"2026-04-23","notes":null,"pass_4":"0.8849557522123894","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-5 (xhigh)","source_effort":"xhigh","source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"eb2ef2efd4e9fc4d936652fbfeb9a1245353e516fab66e2bd8b6ef059ba9a9b0","result":{"confidence_high":73.50104457099735,"confidence_low":60.56975188917963,"sample_count":113,"score":67.03539823008849,"score_display":"67.0","source_stated_rank":22},"run_fingerprint":"27757fbd35b5c39239edcd4c07901d99b0c075d0bf62f96d96b96d38832c2d67","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ed914d982d15585651733aea27f141eca5fbaaf7607a54e677e2b3cf2486dde2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"61.924778761061944","mean_cost_usd":"5.100437004424779","mean_output_tokens":"31159.49778761062","model_release_date":"2026-04-23","notes":null,"pass_4":"0.9026548672566371","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-5 (high)","source_effort":"high","source_model_version":"gpt-5.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"768aa5ddbea0787780aaf541cebbed09a87e44f8dedd53465b3a5999462782ae","result":{"confidence_high":67.4973736229568,"confidence_low":61.263688323945864,"sample_count":113,"score":64.38053097345133,"score_display":"64.4","source_stated_rank":29},"run_fingerprint":"66a83183a2f8297585701936cac931aeb44a041aca496af00b355c76efdf8adf","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6e77f68010bcdd1c3de665580fe9396ab9e854e15cba575fa459e889d93105dd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"45.982300884955755","mean_cost_usd":"2.749342734513274","mean_output_tokens":"19625.433628318584","model_release_date":"2026-04-23","notes":null,"pass_4":"0.7787610619469026","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-5 (medium)","source_effort":"medium","source_model_version":"gpt-5.5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"1dcae931ad56580712462e4bcac8901d605fdb0c0394ef262ee248baa463b28e","result":{"confidence_high":56.53538838024673,"confidence_low":51.42921338966477,"sample_count":113,"score":53.98230088495575,"score_display":"54.0","source_stated_rank":40},"run_fingerprint":"32c8b5381d271811e954814edd86e2e9e72d861b223d7e585a65362225ba0538","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ba603612452c572ae809c78607a9b625afcd67e880fcc9c0c7f3d022dcf134e6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"28.06637168141593","mean_cost_usd":"1.2002025442477877","mean_output_tokens":"9442.774336283186","model_release_date":"2026-04-23","notes":null,"pass_4":"0.4778761061946903","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-5 (low)","source_effort":"low","source_model_version":"gpt-5.5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"8fc6ececc37e100797867a08a59e1f4745e242463825b87d29f36108b1dcaac7","result":{"confidence_high":29.285653664485054,"confidence_low":24.696647220470695,"sample_count":113,"score":26.991150442477874,"score_display":"27.0","source_stated_rank":65},"run_fingerprint":"fc5fb4f1c20810468fcbfb66339beeacc29cb531f9214ebf5693f969a5541950","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_e361abe5266a3ddfdcef017b","split_or_window":"openai-provider-comparison","unit":"percent","version":"DeepSWE v1.1 · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"70b96d52e5689cd34213bff6292bc1a6ce20bbbd8f36e348d2f002209ee83360","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · DeepSWE v1.1"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value for DeepSWE v1.1; competitor rows are secondary re-reports; exact scaffold, task coverage and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"coding-agent tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"28bd89a57c6784aa44dda4f48bb1ba1e048300dc4313759e24d750d50b4e533a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.0,"score_display":"67.0","source_stated_rank":null},"run_fingerprint":"121c2f74665d145ae76865676ddbcfb1c1f5207cbbc4318d1d05be1010e7e252","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · DeepSWE v1.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0ff972701cf87de48b97996","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_7664765571d731e0f6ff7adb","split_or_window":"openai-provider-comparison","unit":"percent","version":"SWE-bench Pro · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"9d36ab983559020174ad87cc6e3508661f012d79c12f52c7527db27d1e23107c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · SWE-Bench Pro"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; OpenAI rows are provider results, while competitor rows are explicitly retained as secondary re-reports; exact scaffold, task revision and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"bb784ac6361cbbe179556c67a1258b8773e5fdda82afa9a06e9f85b5d8df29c2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.4,"score_display":"59.4","source_stated_rank":null},"run_fingerprint":"d774fafcf096d72aa5cead6e2f5344f4bc7c938adf97485cb65609a75a2a1ead","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · SWE-Bench Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0ff972701cf87de48b97996","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_50545b611e994b43926cee37","split_or_window":"google-single-attempt-comparison","unit":"percent","version":"SWE-bench Pro · Google comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":165,"at":"2026-04-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"7c782d7f22e4d709d200e6d16191dc74f6b3c51caa7e59260397198a740f6d37","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · SWE-Bench Pro (Public)"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google DeepMind provider comparison; single attempt; exact scaffold, task revision and thinking level not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-23","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Google comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"fc634252bdb719eab4c7ef57ab98380890e4dcfa72d53ad1fb8194c07a8a5b45","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.6,"score_display":"58.6","source_stated_rank":null},"run_fingerprint":"6b194b6f43c6d019deb493d20c41ec6cc9dba406416af8375a22c35ba7e8d896","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · SWE-Bench Pro (Public)","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9c9546ee249e994755683b0f","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"0f3d91daec09494e65bce94ba3b9253953e0f811ebfef5b194860bc8c14e828d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.85","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-23","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"32","tokens_per_task":"17534","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Extra High","source_effort":"Extra High","source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"313d1baf29c0d69e4ea6a4ff6835c23ae224f579a7fa161bdb739c7fe8c5b0b7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.4,"score_display":"58.4","source_stated_rank":40},"run_fingerprint":"2d8ac0f2f990ecd3a56f5d49388ca7cbfe4999f80e0b8f91431a9a3444780ba1","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"c1a68a12d801ddd647f4b9a8ace70d3a3327cbf678f772461cf21aa7bb1d6d9f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.05","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-23","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"28","tokens_per_task":"12183","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 High","source_effort":"High","source_model_version":"gpt-5.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"bd90c807d496d7e1d285d6e01cd5b56ea0c59abf1dff570e4bcf7b39b12f4f67","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.4,"score_display":"58.4","source_stated_rank":40},"run_fingerprint":"c617d6d5de391f6cae4fb762feea1dda70503c5ec586928a3ec14cfd1816c576","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"3ca55c5c34b136753d5be07d76a517d9fd1dba4d5138cc0cfa5459ad851689f0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.51","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-23","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"25","tokens_per_task":"8522","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Medium","source_effort":"Medium","source_model_version":"gpt-5.5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"88d607cfe2da830471cd4ebc6b409e6b52ed1a07cbd26089ff53f9dd48b9868b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.800000000000004,"score_display":"53.8","source_stated_rank":50},"run_fingerprint":"44009ee6d64b6e8b02d334a24953c467ffdc6e1de5718b8ac732f07b7c714911","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"7c7c1f1e2002e2c3e2ca3f4981e370c869218d8267f764fd860fcab9b2cbebd9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.98","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-23","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"20","tokens_per_task":"5168","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Low","source_effort":"Low","source_model_version":"gpt-5.5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"b9dad61c4dd4b61aa7563ced671a3b14e7308a070e52ab013c0eb847a948a352","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.6,"score_display":"46.6","source_stated_rank":71},"run_fingerprint":"bcde15e4a886e8904087e972f97ad1ab7b6da8ef4d70f7f59f128553115cf3db","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":161,"at":"2026-04-27","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-27","status":"stale"},"measurement_id":"2f86599b084ab16a689aba585fba2198e396cadda2354502832a57ef0b1d1248","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":"2026-04-27","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"415724002e0b1b17ef3d982a693cab73330cdb784e569fd3c433322353fe9600","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.2,"score_display":"40.2","source_stated_rank":5},"run_fingerprint":"7c56cc9c94b8dce93ed60266d77138230ec49ec5806949db10bd39f421a259c4","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"14e116ca82929077125a5ee95c6a14fd6d84148cd23ef9fa4da36b96713cb971","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5","source_effort":"xhigh","source_model_version":"gpt-5.5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"codex","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"a5eefc6dd41a3da732d191b44e47210b280e822c872fabb8a4c49fdbb88b3e20","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":42.96,"score_display":"43.0","source_stated_rank":15},"run_fingerprint":"4de1faa8d1f4af0616d9f4f06bd72b205e6aa8ffffb2485415bf2e61ebebf00e","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"443278cd93e5fbbf308a9c7b5be62b62e4263ae2badd081994e6dff56d9aed8a","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.0666,"date_is_proxy":true,"latency_seconds":166.845,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.018},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"e920434ec568acfaca29619dba9cb98e25c98d70ae0ba8fee4830dffde251aaf","result":{"confidence_high":87.29128,"confidence_low":83.30072000000001,"sample_count":null,"score":85.296,"score_display":"85.3","source_stated_rank":35},"run_fingerprint":"19f5dec349e47808d83858961223fb0faab50c4fb9c65e19f252e8ae27115ea9","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9519d2d177cc35c359df02160a1b7d429983bc192a0afe97b03eaec2829c2adf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (xhigh)","source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e29e6c6333c875441ee487b9b6ce649fb53b2438ce7e884afb6b19713b37b920","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.1342592592593,"score_display":"56.1","source_stated_rank":33},"run_fingerprint":"a6a67005d52ad705b5b0a5276e441cb659f1a85abedbfd6871752f8f135c5557","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"516a2d7c9e257a288b14a9318a00e439209c474715fa56fc087d3a4170c18fa3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (high)","source_effort":null,"source_model_version":"gpt-5.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"96fa133df2d82026f27c68deba84092fabd2cd765a0f76ad85a9be6b94e7d460","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.9027777777778,"score_display":"55.9","source_stated_rank":35},"run_fingerprint":"efdb21222240e4618f597c14d5475b1e567ef333c5e8fb2ebf94e55c93b8041e","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"56f79795f77c48dfec4b41d6831042505869fea56930eadd7e19625d367ea71f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (medium)","source_effort":null,"source_model_version":"gpt-5.5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e90f0e8cf527d8aad7728e46f31d0ea7fd58a7c37bcb58408989a1d46c01e4e3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.4722222222222,"score_display":"53.5","source_stated_rank":62},"run_fingerprint":"d594703883f228b99c6ce0bf2ed430d4f90c16192115ac837104e7597c4b89cb","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"12f77ce25aa6d094d8049d1d7d620e019f7d185724aa26a31fbfa448e912f302","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (low)","source_effort":null,"source_model_version":"gpt-5.5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"d480d5f74f68853e87224d23fccc800371835f4e6f61c7c1a01e9a1d3f0639d7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.6203703703704,"score_display":"51.6","source_stated_rank":75},"run_fingerprint":"02ef441083dba1c8c59c77207ea3788e1c2b3d34347d924b4eda86816d9c7a9d","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"86538b0710cabd864c1d66149f90aef9cd837d3257c5505b103065df3d891294","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (Non-reasoning)","source_effort":null,"source_model_version":"gpt-5.5_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"494afb57ae2b0ed8aa7159de37e811f6d2320d6c938aed4be875d6b518136756","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.337962962963,"score_display":"47.3","source_stated_rank":107},"run_fingerprint":"449422483e4da94565b5bf89af4f55a17bc32a6ab69ab06d8dc88564acf6a1f9","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_528b7e4d648d2d13b940d84f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3f1e0c2abb8693c723293fef3084b8a75b9821641a53440ebe2e37ad353402a7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"1.517132","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.5-xhigh","source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"97b8488dc2ad8f8c25dd8467ee04388186fbce1e627f2e51de5c4985a0e11454","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1942.97,"score_display":"1942.97","source_stated_rank":9},"run_fingerprint":"5c19d43ada316d23fa0e431afe2f2a12e3429d83ce5340756345f341dabf8aeb","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aefaf7964dc2616f3bedc771cd7f3102b2cc16755ecb7b4228ff444509a1c805","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.328965","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.5-medium","source_effort":null,"source_model_version":"gpt-5.5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"2ae49e7a4fb1e8de602baeb3887a1f718c4fb92f2bfb6f80a0e707bda9489c57","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1589.38,"score_display":"1589.38","source_stated_rank":14},"run_fingerprint":"6858c9ea8a5fdafc4780f4838c39e3373a63615c30d67dab68e3b8b44b079968","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a23bde2c384d9f2febbc866fa71d468834688af03a9d1e12856d7c3b0616c43e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.073484","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.5-none","source_effort":null,"source_model_version":"gpt-5.5_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"fc93bf489097cfd637f297f5231907d8295e8a377a9097add7efb24b9045c06a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1127.58,"score_display":"1127.58","source_stated_rank":42},"run_fingerprint":"8b85ff4516c7651f7b096c6fa0531ef7f8be825ee009e2e57dbc8aeb48384581","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_528b7e4d648d2d13b940d84f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":165,"at":"2026-04-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-04-23","status":"stale"},"measurement_id":"1b853761a27cd7706e67ae1f72958a306d55512f56579596fc1499187705c063","metric_details":{"benchmark_page_updated":"2026-10-02","cost_per_test_usd":16.661911,"date_is_proxy":false,"latency_seconds":1911.567,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/openai_gpt-5.5","source_locator":"Results · Overall accuracy","stderr":4.535},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"ed8727e2922f44dbad0ad33ad9f235e52fc271d57670fb0886703a1291267120","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.5"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"91acee21e8c25a80b4018ef3bb0b6d68747b4c809341a3e852297364b81c155e","result":{"confidence_high":78.73559999999999,"confidence_low":60.9584,"sample_count":null,"score":69.847,"score_display":"69.8","source_stated_rank":36},"run_fingerprint":"8a1887574a26546c1274997fecee4d8e9f12537f2f807d42c00dfaa6a47f0fb6","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5150c5880bf45c0a2974ba453e4e9083ffb2bb2f0972ddbeb547773f7aafc8d0","metric_details":{"license":"Proprietary","variance":9.16109633481052},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.5-xhigh (codex-harness)","subset":"webdev"},"run_count":null,"scaffold":"Codex harness","tools_allowed":null},"protocol_fingerprint":"2e8829545c92eb08e662f3619392497479e601002735a32f27f629eb348f40ce","result":{"confidence_high":1518.2243183645855,"confidence_low":1506.3597535899996,"sample_count":15873,"score":1512.2920359772925,"score_display":"1512","source_stated_rank":50},"run_fingerprint":"157a4cf459dfa687dfd2d1963e3d1f35e14aaab791daa25f7a9d4543f0d2e9ca","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"74af152291969b1681b651666b8a223f78374e54d31081a402a14a1ac9545414","metric_details":{"license":"Proprietary","variance":8.984166587116096},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.5-high (codex-harness)","subset":"webdev"},"run_count":null,"scaffold":"Codex harness","tools_allowed":null},"protocol_fingerprint":"6689241c49cbe5765e115582c03b1c61189bf5fbedd5a336df85aa228b99c28d","result":{"confidence_high":1492.893055931945,"confidence_low":1481.143620884648,"sample_count":18174,"score":1487.0183384082964,"score_display":"1487","source_stated_rank":57},"run_fingerprint":"399dd97b026e4e79334153e0a78ce0af3e8fdf2ae5cb1048b1e21b9d8bccdeb0","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c609b8385a4561f0daff82530d037a007bcd6a9f17c4e101aadf59a189a6f3f6","metric_details":{"license":"Proprietary","variance":8.935971773412884},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.5 (codex-harness)","subset":"webdev"},"run_count":null,"scaffold":"Codex harness","tools_allowed":null},"protocol_fingerprint":"97bc2c63db458f1ba07043131b4ef8d4e234fa910539f6adc813151d6c13131c","result":{"confidence_high":1461.559494277491,"confidence_low":1449.8416160411462,"sample_count":16061,"score":1455.7005551593188,"score_display":"1456","source_stated_rank":66},"run_fingerprint":"b06ee67ef1d1a845fbfca7677b895350af2bd6a9738d5c4082b7f5d3eeaa7b03","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a53012d563cde83034dd51a0","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0cb4b6e2d70486ec0cde75fea271c7e6d9689b6272c589df9de2089aaa7503aa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"d92af3d0d0c429d2b1a632e44c6dec9231747bfeb0e34c44605fd5e4b6ebf7d4","result":{"confidence_high":1516.23,"confidence_low":1504.06,"sample_count":14946,"score":1510.15,"score_display":"1510.15","source_stated_rank":48},"run_fingerprint":"b0ee5f5ed87f45d525e46bf18ef82698652355e777fa65974e9fd3b06594c2ce","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"826ee9fac54497135bd1dafa73a6a47c600042f2ba74a1a3010fe576777c8701","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"8b84c041735a6a779811f9e997f0fcd6f3829ceb6a005e8daf6f40227496638b","result":{"confidence_high":1493.01,"confidence_low":1480.95,"sample_count":17020,"score":1486.98,"score_display":"1486.98","source_stated_rank":55},"run_fingerprint":"519acaeb253e13a6beb20f14a57638b81ec205236dea1e8467b8065699c1f7d2","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cfb2e2e2584fec97fa33f4e9a42ecefb3dd593957e3c49157c14f76cd6490ae4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"ed4271208426e1f48698aba5eeb9c4ea8ca8244a0aa4ef425ceae27649e4cb45","result":{"confidence_high":1463.81,"confidence_low":1451.72,"sample_count":15012,"score":1457.76,"score_display":"1457.76","source_stated_rank":66},"run_fingerprint":"b1ffd0b6419cb69e60f5fa6e8d1e215c0bf0b2a31618b7456a290b51b1266a16","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_328b472e0ef57c1dc4b68276","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"mirrorcode","metric":"mean_score","name":"MirrorCode","release_id":"release_07bf24088c0805f7b6df7a88","split_or_window":"15-programs-30-language-tasks","unit":"percent","version":"ML +Private 2L"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T16:12:45.865Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T16:12:45.865Z","first_observed_at":"2026-08-28T19:57:34Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5abf849253379d8a9b4c5b5263589db6086a852c57f3599b8bc5b2cca9bfaab2","metric_details":{"best_score_across_scorers":"0.1","model_release_date":"2026-04-23","stderr":6.042179781166438},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mirrorcode-ml-2l-gpt-5-5","log_viewer":"","logs":""},"run_count":3,"scaffold":"Epoch AI Inspect ReAct agent","tools_allowed":"shell, text editor and visible-test evaluator; no internet"},"protocol_fingerprint":"c2bfd88c9a14d237e938d328530687bf8542ca96d58446303b33571e8d999098","result":{"confidence_high":21.842672371086216,"confidence_low":0.0,"sample_count":30,"score":10.0,"score_display":"10.0","source_stated_rank":8},"run_fingerprint":"13573db06c41caf99ed8a39b9aadd0af36a41bcb13aaba33060115776b2b841b","source":{"content_hash":"3e212e97e03d0343e676ecc7ff046ee7b94d219ba7d863364634b7361dc89b4b","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://epoch.ai/benchmarks/mirrorcode","id":"epoch-mirrorcode","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · MirrorCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/mirrorcode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"760104b3614d406a7151982d857595d6a6b9c7b1ff18dbebc44ca6417ba82105","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"5f1e963ac2dd299059fdcd04c621362c1517cd74eebd1c8b30c33821cdea8047","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.91,"score_display":"84.9","source_stated_rank":14},"run_fingerprint":"15a02fd03f47c61f7d508e72c694d252f01ee77046f703b24029de7f5382e55c","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"18bc5d88635d1f39d42260864025f6ea4db839940aef5fa101350e336beea2e1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"34d0d66d0784911da57ba1f4d413e8f43590bc5198ee8004f2d15a76c6222030","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.9,"score_display":"83.9","source_stated_rank":15},"run_fingerprint":"01284deba6ba4dc0c63a1b0b0848e024307470b831d61efe68d7919aa2ea028e","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fd32aa57b92aa53f3ae9267f11cf8e5f80c2dfd33d868c032f675fcbb0e85d70","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.5_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"452728bbcae3673c2f62fac2a795e8b45574362bfb9e0a99fa9cf81990af3747","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.15,"score_display":"67.2","source_stated_rank":38},"run_fingerprint":"91ce531f8d88dc097e8d62a42516a7ed1ba02abb5893100f11f822544e5b7eca","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_528b7e4d648d2d13b940d84f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":165,"at":"2026-04-23","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-23","status":"stale"},"measurement_id":"f88ada04d01bc8c421d7bbc3b12f257b223493b93fb31bf04054d85a18eb64de","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"standard_error_points":2.0892351283,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.5","source_effort":null,"source_model_version":"gpt-5.5_unknown","source_row_date":"2026-04-23","source_scaffold":"NexAU-AHE","upstream_leaderboard_url":null},"run_count":null,"scaffold":"NexAU-AHE","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"8ac802e22ba0d10b04a8c3cd385b874758af1fc07e9cf270f7f94b417a7ad477","result":{"confidence_high":88.814001975068,"confidence_low":80.624200272132,"sample_count":89,"score":84.7191011236,"score_display":"84.7","source_stated_rank":1},"run_fingerprint":"93f5b600f4b62f0c0de99fe181d30c595d15c97e4b4ed45784bc52f8e0a582aa","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5124f8d7b9faf07d4e8aa5b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":165,"at":"2026-04-23","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-23","status":"stale"},"measurement_id":"7bc56d568578728035b1110a06a8ad9fc6756fb6fc0feded6c7aa8cf3d23290e","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"standard_error_points":2.1123213002,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.5","source_effort":null,"source_model_version":"gpt-5.5_unknown","source_row_date":"2026-04-23","source_scaffold":"Capy","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Capy","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"a3958dabfc701f9a32dd888de535e0ad210d3ee5829264838dd2e31a524e5170","result":{"confidence_high":87.28621716409201,"confidence_low":79.005917667308,"sample_count":89,"score":83.1460674157,"score_display":"83.1","source_stated_rank":2},"run_fingerprint":"54135219af5f9dd34f6005175b181891a3817dad78a414bb197f9923d9ab9b55","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5124f8d7b9faf07d4e8aa5b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":165,"at":"2026-04-23","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-23","status":"stale"},"measurement_id":"ff30f85059b7be51d20f59f76d4cb8b87575d8011f5bb6c4375b0bd3e974fe0f","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"standard_error_points":2.2241606381,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.5","source_effort":null,"source_model_version":"gpt-5.5_unknown","source_row_date":"2026-04-23","source_scaffold":"Codex CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Codex CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"908273b225cdb7e2874308a8b194422d65deabd5b00a478e0e88086e3bdcc48b","result":{"confidence_high":86.606545861876,"confidence_low":77.887836160524,"sample_count":89,"score":82.2471910112,"score_display":"82.2","source_stated_rank":3},"run_fingerprint":"72438680c23fe259f3319b07800bea6f1bc3570b0efb8c83c79eaa51eacad6e5","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5124f8d7b9faf07d4e8aa5b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":165,"at":"2026-04-23","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-23","status":"stale"},"measurement_id":"97279f2817b36f6886e9dea2a9eeefc2f80779868faada9b7167a6ee1f964259","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"standard_error_points":2.2,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.5","source_effort":null,"source_model_version":"gpt-5.5_unknown","source_row_date":"2026-04-23","source_scaffold":"Codex","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Codex","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"86cfc2f83208fa886e4cb94a493336fa322e8abb035e2b6b2cdab136cc2ce229","result":{"confidence_high":86.312,"confidence_low":77.688,"sample_count":89,"score":82.0,"score_display":"82.0","source_stated_rank":4},"run_fingerprint":"ff85a60f9118e2353665edd39f7c877e7a34b2cc028351f9258629b2091a9716","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5124f8d7b9faf07d4e8aa5b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":165,"at":"2026-04-23","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-23","status":"stale"},"measurement_id":"a9003773e31042bcf700ae4665e28f6ce86553951c863f7a87f4f673bde9acbb","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"standard_error_points":2.4915582762,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.5","source_effort":null,"source_model_version":"gpt-5.5_unknown","source_row_date":"2026-04-23","source_scaffold":"clnkr","upstream_leaderboard_url":null},"run_count":null,"scaffold":"clnkr","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"6f709453765bddcf06c398f7926db49edd6db056358dd5acd307ee8dd35ba914","result":{"confidence_high":70.950869951652,"confidence_low":61.183961508948,"sample_count":89,"score":66.0674157303,"score_display":"66.1","source_stated_rank":32},"run_fingerprint":"1c8202273e418a338063be651e6ac62c737b47c3c3898de4177c0678af2ece99","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5124f8d7b9faf07d4e8aa5b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fca4889cf5a55e49e656d8cde75b7df9f6dc53cf915bc04b204719df5df12196","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.741883,"date_is_proxy":true,"latency_seconds":427.412,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.649},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.5"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"00db90dab20394e75adbfb9aef737ce2895e39e2952e4080114b6a8d3a1387aa","result":{"confidence_high":77.67604,"confidence_low":75.13195999999999,"sample_count":null,"score":76.404,"score_display":"76.4","source_stated_rank":16},"run_fingerprint":"453417115016ff595007f7fa77c2fe82f60bf7d5a655679d3447653a104f528f","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_d522137a2b9b55b4156d183d","split_or_window":"single-agent-baseline","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"27b737461a89484fadeff781335632f2c5bc7a8f7f6a97086ee5b1384b8cb01c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · Terminal-Bench 2.1"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"single-agent baseline; exact harness not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"terminal agent toolset"},"run_count":null,"scaffold":"single-agent harness not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"35ec4343c781d4675f2f64710c5000fe0977214013c6f6f70121275e1c4fe180","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.6,"score_display":"85.6","source_stated_rank":null},"run_fingerprint":"53c32a4e1f5f15d292cfad50b33b2f500be95a10ab2a0f1d5adb0d919a2210a5","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · Terminal-Bench 2.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0ff972701cf87de48b97996","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_39c649809acb869ddd02ba8c","split_or_window":"terminus-2-provider-comparison","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":165,"at":"2026-04-23","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"0c589d091f5f7ba032e0c384dbfc7b813ddfbe3624d7c93eb4cc0fdd5104b201","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · Terminal-Bench 2.1"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider comparison; pass@1; thinking level not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-23","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"2f562581af2e43f1a4516a6befa0908bd789462c520e685eb7882137b21a195c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.2,"score_display":"78.2","source_stated_rank":null},"run_fingerprint":"65b0f13de2f638f2482c85e1b83581d399d30077b5e282d433de5af659571acf","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · Terminal-Bench 2.1","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9c9546ee249e994755683b0f","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":75,"at":"2026-07-22","basis":"evaluation_at","evaluated_at":"2026-07-22","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"954fce5b0be61e12097e8aabf52a50ab8b7b1e2d166e202c1e955f79917bc25b","metric_details":{"cost":null,"pass_2":37.62886597938145,"pass_3":33.24742268041237,"pass_4":29.896907216494846},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"gpt-5-5_sierra_2026-05-05","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"ca1b0f52e5f57f2a197d9554fdac27ff8518c746bdb61a2a89fe030c814a6421","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.58762886597938,"score_display":"44.6","source_stated_rank":null},"run_fingerprint":"eb10c682afc8db74bd90371603030ce23ec3423d24ffcb7f3337ddf29b7d19a2","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":159,"at":"2026-04-28T17:25:29Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-28T17:25:29Z","status":"stale"},"measurement_id":"73e6274447933bb6879ab362c68003ca146b255b3a760813edd9ce6a7188c32e","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.7,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gpt-5.5 (xhigh); createdAt=2026-04-28T17:25:29Z","source_row_created_at":"2026-04-28T17:25:29Z","task_pass_coverage_threshold_percent":75},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"gpt-5.5 (xhigh)","source_reasoning_label":"xhigh","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"113e30cbcb2232483cac3e939d6d3d3afda1dd93b1b4bc0111b0ac2ab754f57a","result":{"confidence_high":78.0,"confidence_low":72.6,"sample_count":1000,"score":75.3,"score_display":"75.3","source_stated_rank":3},"run_fingerprint":"569c46f710218877ed31eedbc6fe02406609b0b47e21752bd00e123cec052a8d","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=gpt-5.5 (xhigh); createdAt=2026-04-28T17:25:29Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"2707b6e576b5cac0a3c6580eebf526a0cd6c1e21b70b1cd7f492d2cae500eec3","metric_details":{"expected_trials_per_mode":261,"normalized_gain":32.6,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":15.8},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":75.5,"confidence_low":59.099999999999994,"sample_count":87,"score":67.3,"score_display":"67.3","source_stated_rank":1},"run_fingerprint":"481eb250f485c83e56664ebebd379a05f19f8fff08010639e58cd7b4680ab523","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_815be32a4499e3fe8b5cd1a6","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"d03e2101db72e7f3867b77b5ee92747e9873307cc3f5b9ac32c7195093eddc9d","metric_details":{"expected_trials_per_mode":261,"normalized_gain":37,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":19.7},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"codex","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"codex","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"de4ee8be7ebbb4075f4b810c6a6dcd2e1f701030bba81db0d7f2320352479107","result":{"confidence_high":75.0,"confidence_low":58.0,"sample_count":87,"score":66.5,"score_display":"66.5","source_stated_rank":2},"run_fingerprint":"5ee672efb49e6c391c8a15038b40caca7ad5a760096a6b8b9e2fdf7097dc7088","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_815be32a4499e3fe8b5cd1a6","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"3339a59746d4e91aede1ce07bcbb0ec089b23edbfc206aad90b4321f95fa26a3","metric_details":{"expected_trials_per_mode":261,"normalized_gain":32.6,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":15.8},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":60.5,"confidence_low":42.5,"sample_count":87,"score":51.5,"score_display":"51.5","source_stated_rank":1},"run_fingerprint":"aed86baf7952c23b917ec15d045e7042c53af9d9e68a4ec31890a9e89e6690da","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_815be32a4499e3fe8b5cd1a6","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"6382e88cfd5ddce04fa8f00a9ae0f4573b8cadc7edfeda81711cc36bcbe21494","metric_details":{"expected_trials_per_mode":261,"normalized_gain":37,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":19.7},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"codex","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"codex","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"fb812031761af2e4bf7d902ad4e16eab19781fbf07e90c9eda8d41c09aed7621","result":{"confidence_high":56.0,"confidence_low":37.599999999999994,"sample_count":87,"score":46.8,"score_display":"46.8","source_stated_rank":2},"run_fingerprint":"165b30f36a3a023ad1947794e5f3ee5f656cb9be21644afcd3626fe9147b1c18","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_815be32a4499e3fe8b5cd1a6","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"b3dd917d2db20ea65fd2030d46fbec62f55a28d4075134e74b3bb2793a254756","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.555","mean_standard_error":"3.6","model_release_date":"2026-04-23","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.8,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5.5","source_effort":null,"source_model_version":"gpt-5.5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"2f7562ad568fe30eb6fed22782891ce28dcd7e3a4ef023b6e32ca4ab93b08093","result":{"confidence_high":45.948,"confidence_low":31.052,"sample_count":480,"score":38.5,"score_display":"38.5","source_stated_rank":12},"run_fingerprint":"cf5da652dfc10246dd17104b41eb1e0f4c004f0e90ef4f2d1e58dcebc12fef23","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f724b71446015637ed1043106875f0edd162fdccbdec990357b70de9ef72e43c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.703","mean_standard_error":"4.4","model_release_date":"2026-04-23","notes":null,"standard_error_points":550.0,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5","source_effort":null,"source_model_version":"gpt-5.5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"065aaf84ed4cfccb8ba6726725005441ba8fe5169824f67c84aee2c568d1609e","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":55.1,"score_display":"55.1","source_stated_rank":16},"run_fingerprint":"cf93fdab3541a56f0e833f03939425ae5650af1825267546ab827a7e34b6e2a2","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_aa4c4f569e5bcfdacbdd9f30","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"337d9c5b03ded9c3fc9bbcc0ec5d23b8830e936715c45f0da3e19dadd6bea8f1","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":55006},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 5.5 (xHigh)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"841028e3322929b7b833effff78d7535b59229bd58d5ade81cbb59cfb8233602","result":{"confidence_high":0.05179446341627249,"confidence_low":0.03129060767491741,"sample_count":3060353,"score":0.04154253554559495,"score_display":"0.0415","source_stated_rank":15},"run_fingerprint":"d07bdc862418fd2db0b9f696ec7fdbbae27996b941cc10fece2873db80c690a0","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"402dda50b0e9f8e8400e9623e01c775d171f654c0a29f28c512f332becb1c2d5","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":82416},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 5.5","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"56ccc235c892d82ecc4d230f999ebd22b04627db5a804451d2e0da85940626fc","result":{"confidence_high":0.020171544455983667,"confidence_low":0.0006095469753625207,"sample_count":2659460,"score":0.010390545715673095,"score_display":"0.0104","source_stated_rank":29},"run_fingerprint":"39db41f194a0e360e1e3484a065515ed2e1788ea41468a0fa6955b8802ee1389","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d22b7d047c50a44ce81761d2","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a940385dceef1d84dfdc3b1f6a8a3d9597d3e50c10983aaabfeaa43708515566","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5","source_effort":null,"source_model_version":"gpt-5.5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"16dcef846831a895f1270680fca37a67cd20d43510a0e7fceeb421430b42eb58","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":7523.843999999999,"score_display":"7523.84","source_stated_rank":11},"run_fingerprint":"6c4185d2c64c93660808249e78e6536f5475d9a1ca97ebb0eb108830f58b5c2c","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e2ab7970e7428fde1eca209f","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"161630ff36bba0884a65d5a36e8133f795e59ab6ae53b1284e2f9a66f1496336","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":922000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-23","reported_confidence_interval":"-21 / +21","source_locator":"Leaderboard models table · GPT-5.5 (Xhigh) · gdpval"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-5","upstream_model_label":"GPT-5.5 (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"5d6e0009dbef93c77a7552e1861156950fd5edba8c09b3e3086af74239f94d2e","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1335.89,"score_display":"1336","source_stated_rank":null},"run_fingerprint":"121dc8d2a51499a2e0a4763c9ad96fe0ab739e0be04dfad3166ca349a12de07e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.5 (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"26912e117170ee52d6402984741d47f628c4ff5f4f2a196de0df786c80a91a5f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":922000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-23","reported_confidence_interval":"-17 / +17","source_locator":"Leaderboard models table · GPT-5.5 (High) · gdpval"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-5-high","upstream_model_label":"GPT-5.5 (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"0ae7b730305fa28cf268c0d29f990b878f754171ffc1f74a8034c56dff4ec1e2","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1329.25,"score_display":"1329","source_stated_rank":null},"run_fingerprint":"cf280dad01c19e4a376ca5b0e27a84b9e9a07a5d2b8e2988310d9aab354bc771","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.5 (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_de975f255b440d53e37ca312","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9f1847fc15ed713f713ae4752cd24063c8d7502e4c3dfc713749ef65792442b6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":922000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-23","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-5.5 (Medium) · gdpval"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-5-medium","upstream_model_label":"GPT-5.5 (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"ed2bd488dc5da7428328fd9ad5c15c2ba54fd6414371c0932d184f9905ec4fd0","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1218.43,"score_display":"1218","source_stated_rank":null},"run_fingerprint":"db28d975a5ef6ff2df23d31f65a220a97ab72b2696d966d330dd6b163efe72cb","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.5 (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_f669e0173f71ff9f02fbaf9b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2c8be2b2acdec68898963d550523f54c247886b5a82e886fdd7fa8a89158ce24","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":922000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-23","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-5.5 (Low) · gdpval"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-5-low","upstream_model_label":"GPT-5.5 (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"d1636d4a9696f65e1bed29ae9d92903516b837ec21073070f1694ce030146eba","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1032.12,"score_display":"1032","source_stated_rank":null},"run_fingerprint":"337f543a095fb91fdc8a98a5cb331cc1f9a6f07ecb280dcbe95a185288100409","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.5 (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_aa5bdf7c68b8942c3fc5b1ea","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bddb841ac11546ba4bfda04703032d109a918648ab4b53950fb55bbd4cb1ed36","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":922000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-23","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-5.5 (Non-reasoning) · gdpval"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-5-non-reasoning","upstream_model_label":"GPT-5.5 (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"e86cff541edcddd229ed3c4b86b36b8f61cf7d1ee92facb32c5a7c5f32c1f045","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":957.06,"score_display":"957","source_stated_rank":null},"run_fingerprint":"144c759ee422900614023bf7e1c27ea85a389e352883a2f74ae6dcc7d7bfd9f9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.5 (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_528b7e4d648d2d13b940d84f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"679278325a868f018619a0398beaaf95a4c46b5c3a5dcbe80cc7284528633364","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-23","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"openai/gpt-5-5","name":"GPT 5.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 (xhigh)","source_effort":"xhigh","source_model_version":"gpt-5.5_xhigh","source_row_date":null,"source_scaffold":"batch tool","upstream_leaderboard_url":null},"run_count":null,"scaffold":"batch tool","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"a35a7e4f1f52502187e775bbdee0cee04878e831ca9612df4ba9c8ae53cebafb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.0,"score_display":"13.0","source_stated_rank":11},"run_fingerprint":"a9a35ad0acf6c28126869ba7a4344efcd9de2b74b616d1a714f442298b2cb8f3","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_618796ded36fbc2ed30584a4","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6a4e4f3e18754a338c216498d0efd863232b7c4f7d541f860c6d51ca8e725126","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-03-05"},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.4 Pro; model release: 2026-03-05","source_accessibility":"API access","source_model_name":"GPT-5.4 Pro","source_model_release_date":"2026-03-05","source_model_versions":["gpt-5.4-pro-2026-03-05_none","gpt-5.4-pro-2026-03-05_unknown","gpt-5.4-pro-2026-03-05_xhigh"],"source_reasoning_efforts":["off","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ac75386b294a04bea4f90d4b0c4bd7d8ce6616d45fd1c8542305f18a67db6310","result":{"confidence_high":161.92,"confidence_low":156.47,"sample_count":null,"score":159.12,"score_display":"159.12","source_stated_rank":11},"run_fingerprint":"245b5e30c2281e0eccbaa2ec7e4714435e64b01836ad91c621051dd5989c0f15","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_810ad01488e903e7ff5a14ea","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":195,"at":"2026-03-23T21:12:56Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-23T21:12:56Z","status":"stale"},"measurement_id":"209c937f5bc556e362d7806dfacc639df4922f902c8c9f040394562e94ea1fda","metric_details":{"calibration_error":38.0,"confidence_interval_half_width":1.95,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=gpt-5.4-pro-2026-03-05; createdAt=2026-03-23T21:12:56Z","source_row_created_at":"2026-03-23T21:12:56Z"},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"gpt-5.4-pro-2026-03-05","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4e3ac3d26643d4f2fd6f4b37a5ded1ff80b5168a868b8225fc4734f6d48168c6","result":{"confidence_high":46.27,"confidence_low":42.37,"sample_count":2500,"score":44.32,"score_display":"44.32","source_stated_rank":2},"run_fingerprint":"a63d9e3d79fe7ce7d5919308c242a33f28d37eb091ca0d157d81761faac8bff0","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=gpt-5.4-pro-2026-03-05; createdAt=2026-03-23T21:12:56Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d9016c6789c93f7997fae173","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:28:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:28:24.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ec53abbffaab1c0f84500bd14dd2b3a4f58cbe3da8e8f1a5d127c0fb54b5465e","metric_details":{"best_score_across_scorers":"0.463","model_release_date":"2026-03-05","stderr":1.5775927227262425},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"RfSJbuuuMkeVHQj7xrth4u","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRfSJbuuuMkeVHQj7xrth4u.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/RfSJbuuuMkeVHQj7xrth4u.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"7a9312bb814bd62a83ca8daba17fab3f15dd4cacf4f0f7d654c6220ef25f94ea","result":{"confidence_high":49.39208173654344,"confidence_low":43.20791826345657,"sample_count":1000,"score":46.300000000000004,"score_display":"46.3","source_stated_rank":36},"run_fingerprint":"4191b87e3d3858a17386339f61ff58899d50edcf4d3ce0a9ee0df57e8dcd04d1","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":199,"at":"2026-03-20T00:19:39.820Z","basis":"evaluation_started_at","evaluated_at":"2026-03-20T00:19:39.820Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"dcb835fc70f43e10f4a89e9e8093d3bd92374ce9abaeb897af73840e32d12f3b","metric_details":{"best_score_across_scorers":"0.946","model_release_date":"2026-03-05","stderr":1.6},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gpt-5-4-pro-gpqa-cloud-run","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e472fbb92aa2c73008c634b78c57262b905e36573196b7808f984bba5edb23f6","result":{"confidence_high":97.73599999999999,"confidence_low":91.464,"sample_count":null,"score":94.6,"score_display":"94.6","source_stated_rank":6},"run_fingerprint":"20cfd8a81558eac9899bfdb5e6ef2abb60e59279422f4ec9746320484474bb3b","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-13T02:38:02.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-13T02:38:02.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"feca71efa7fb2ea5774953400bbcf67ccdcd59cbde97df8fd7358ca1b417cf2d","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.8245614035087719","leaderboard_private_problem_count":285,"model_release_date":"2026-03-05","public_example_count":10,"stderr":2.2569134425265194},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"ZFVa8dw7oiir6PhcF8XZMo","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZFVa8dw7oiir6PhcF8XZMo.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZFVa8dw7oiir6PhcF8XZMo.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 Pro (xhigh)","thinking":null,"upstream_model_id":"gpt-5.4-pro-2026-03-05_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"c8064973eb5564782e385b99cac1d2a5363d6786ade312eca4d6beca296ca1ac","result":{"confidence_high":86.87969069822917,"confidence_low":78.03259000352521,"sample_count":285,"score":82.45614035087719,"score_display":"82.5","source_stated_rank":13},"run_fingerprint":"2acf3c30e637a389b0272cab27c2810c4d8e86d4de2e9706961ecb384b8cf6a6","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c7d8599d11331bdbed846ce59163f351626b5ed7f04d9319d3751ee8797daa7b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":16.41,"model_release_date":"2026-03-04T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4-pro","model_type":"CoT","upstream_model_id":"gpt-5-4-pro-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d70d8bdb8ded770b2df917285d64d3028140c60dbda01cec10ac66a81e6c4e43","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.33,"score_display":"83","source_stated_rank":35},"run_fingerprint":"9abea9196bf854e3a011dd446bb52dfd08a4ee3385db5daa426752795393866e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fd1f4040d562e05514d80cc7aecd56a29931d9ec36fa3c8d8ad3ee4e1cd3a068","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"16.41","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Pro (xHigh)","source_effort":null,"source_model_version":"gpt-5.4-pro-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5ecc1ee9854ae6e3e1edd60a0c40b6b2dad5e70da19b082f213eb9d638d83f76","result":{"confidence_high":86.823954,"confidence_low":79.132221,"sample_count":360,"score":83.33,"score_display":"83.3","source_stated_rank":37},"run_fingerprint":"87e57e76bb869808cbdc358147a2b13a96bd9a6292b8f75931964b576f3fb1c9","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"63cb1c2f12d4560864fc4901497685c715ba7503b79f463a7830b575204cb648","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Pro (XHigh)","source_effort":null,"source_model_version":"gpt-5.4-pro-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"59a3b30e42cdad1ae135e69c543b789f02d0827c24f139096cb5f6d223c1f7f0","result":{"confidence_high":86.823954,"confidence_low":79.132221,"sample_count":360,"score":83.33,"score_display":"83.3","source_stated_rank":37},"run_fingerprint":"b8b66bca1ca6c354eafebcb4e02045b6234af605c0053feb705abd5269e634a1","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-13T02:38:02.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-13T02:38:02.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"7edb0a0713842b61b81f73ca72c1925adaa7b71ca285e1b07253783ad9e30b02","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.5853658536585366","leaderboard_private_problem_count":41,"model_release_date":"2026-03-05","public_example_count":2,"stderr":7.789619230571372},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"W6b3C8AeNC7CAFs5DQma8N","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FW6b3C8AeNC7CAFs5DQma8N.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/W6b3C8AeNC7CAFs5DQma8N.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 Pro (xhigh)","thinking":null,"upstream_model_id":"gpt-5.4-pro-2026-03-05_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"cfcfd105753964b6092a611d388de2bd1d1b6f3f4e2ad588abccb20d09c05eba","result":{"confidence_high":73.80423905777354,"confidence_low":43.268931673933764,"sample_count":41,"score":58.536585365853654,"score_display":"58.5","source_stated_rank":19},"run_fingerprint":"cd904fecd2e5c76afba43a4d17dfb5f3a59061f631f0702d5891eb93daf27591","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"afed9d853da75c40bdd990b2c00450edcc39780b925eb381e21555f227106d19","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Pro (xhigh)","source_effort":null,"source_model_version":"gpt-5.4-pro-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"993fc79ddcee0132a480592dfefb183f7d7c39ca4bc616ba1d843879d02ec27e","result":{"confidence_high":41.45954,"confidence_low":20.593649,"sample_count":71,"score":30.0,"score_display":"30.0","source_stated_rank":15},"run_fingerprint":"2b133af216a871e903a277f3967b231a0e7fa9b041ec620388614505a2c091ee","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a8e4f2241cf21895af3b732f2e8edf08873e6e4788bcc417581d563634f5c0b1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-pro-2026-03-05_unknown","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"8f34d561d4c9ab49272d860affee75cf2fd058ae445783d63eb3ccb55e8e7f12","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.1,"score_display":"74.1","source_stated_rank":10},"run_fingerprint":"7fa3561b7a8c77132056804a68c3dc9c3badd2e7a38c7ec951bfa98dfe9fc8c8","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d919041bb5adc55760362ecf","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":199,"at":"2026-03-19T23:54:02.989Z","basis":"evaluation_started_at","evaluated_at":"2026-03-19T23:54:02.989Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3666493b687d61d5296956ea34a7e4ec70d0c6aa51680a892551724fb4438e54","metric_details":{"best_score_across_scorers":"0.586","model_release_date":"2026-03-05","stderr":5.0},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gpt-5-4-pro-chess-cloud-run","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b3e499595f9fbab4144cefec4d5fc70b2d52a462373c4f3e30332343973cee84","result":{"confidence_high":68.39999999999999,"confidence_low":48.8,"sample_count":100,"score":58.599999999999994,"score_display":"58.6","source_stated_rank":6},"run_fingerprint":"40b426777225ed64b579d5ef23bfee789fdd7c726b1bc7def05310cfbc4d7e62","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"df1de938e76a69a6cacebec6998ca09e07f862b19e460f1a5a3faa4450d9e590","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"7.21","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Pro (xHigh)","source_effort":null,"source_model_version":"gpt-5.4-pro-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d3620fdda0fa9df1c5a9409fcc1a63d89656463398c9dd19a03fe44da9701675","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":94.5,"score_display":"94.5","source_stated_rank":37},"run_fingerprint":"17124604bf2d298bab22667ea79ff3d4104a40907589ff817f7cf29db47a7134","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ae79469dfe326ecd142d150ece4cf99238e228c402692174b52d58ef0aa89976","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Pro (XHigh)","source_effort":null,"source_model_version":"gpt-5.4-pro-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"74b44cd9661887162073a56b779ec09501b776a3167d98bdc8ee05f7ca64f8e5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":94.5,"score_display":"94.5","source_stated_rank":37},"run_fingerprint":"a021600001033ca0633a4ad6ca8f2d4cd0ea4fbfdaf0b43f76b1c5ddc9d8737f","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_b5490cd7180da4cea2102329","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"af4e9508f102df8d2f4a4bfeea8a22ae06b148888ab842e56383454fc7cc21c0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-4-pro","name":"GPT 5.4 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-pro-2026-03-05_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"24f3d247a66f5abc9d7b5ec648f3fddb530e8917ded832cce96c305b839ac051","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.440000000000005,"score_display":"57.4","source_stated_rank":54},"run_fingerprint":"bfd544ee2b2100942883ff92438ccf3dc233d3e1594524c3289d662dd9a7eb3e","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c2e76640622cf0a45f5bec51","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6c4e999a681591f5dd774a463c710bc1ba6cbe11ccdbae68b86a49538b394dd0","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-05-28"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Opus 4.8; model release: 2026-05-28","source_accessibility":"API access","source_model_name":"Claude Opus 4.8","source_model_release_date":"2026-05-28","source_model_versions":["claude-opus-4-8_high","claude-opus-4-8_max","claude-opus-4-8_none","claude-opus-4-8_unknown","claude-opus-4-8_xhigh"],"source_reasoning_efforts":["off","high","xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"366e130c418357323ef95faef49cc81e6ce82169105a1f23b305443bfa61781c","result":{"confidence_high":160.74,"confidence_low":156.08,"sample_count":null,"score":158.34,"score_display":"158.34","source_stated_rank":12},"run_fingerprint":"9bd480c57ac7bf5aff10564c02d6283e38d3c782f79ee09356a6360ddee582a9","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_19fe5fa33d354d1d1a022388","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bd763bdc4c5d148fdd7a89599940d263d4500dc8090815864db6ed4606d578d7","metric_details":{"license":"Proprietary","variance":3.9099509523769886},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1485.385863082314,"confidence_low":1477.6347559163694,"sample_count":63974,"score":1481.5103094993417,"score_display":"1482","source_stated_rank":24},"run_fingerprint":"aed3f9c674c86e470d72f804bee0c51dbc1c13ff829be611fe7e10545bbc6fed","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6e5a68a5e81e10e06d817415f1e8e813e0f65f0079f07bd3013c52367a2d5980","metric_details":{"license":"Proprietary","variance":3.8606193798803967},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1478.3942728418351,"confidence_low":1470.6922184742823,"sample_count":65072,"score":1474.5432456580588,"score_display":"1475","source_stated_rank":37},"run_fingerprint":"e329663343ef5bae4f0a336440fcd276d4ac1e30e64016b058bdec915e75c08c","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d2ca75d5dcbca2f9c8bf8b95","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3ea2c21163f11f603c9d08d609296d5d9803597941b3598dc2e49892b3791f56","metric_details":{"category_scores":{"Agentic Coding":50.504999999999995,"Coding":81.828,"Data Analysis":66.03333333333333,"IF":72.0335,"Language":79.65933333333334,"Mathematics":94.31649999999999,"Reasoning":89.19225}},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.22398809523808,"score_display":"76.22","source_stated_rank":30},"run_fingerprint":"52f20946aed6be3316452c19fe85dace30ce1af87ba03b09fae35ce002314f1f","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-28","status":"stale"},"measurement_id":"b082396918f198ce2501b24ea8e2b8bad459f88e9f6aaeba9513c30b48eef83d","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Table 8.1, p.194; §8.10.1, p.202"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":"vendor-reported HLE grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; 1M total-token cap; no tools; no context compaction","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-05-28","source_content_hash_method":"raw_bytes","source_content_sha256":"0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f","source_published_at":"2026-05-28","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8828394c5ab348b4e326438defb67fbb61251e8435b7c24d33f97db6f8d78421","result":{"confidence_high":51.758787,"confidence_low":47.841827,"sample_count":2500,"score":49.8,"score_display":"49.8","source_stated_rank":null},"run_fingerprint":"1810b710761c27cc29697c1b0fe2e6067b5a83a11a8f3b4241f2c3a9be7d4859","source":{"content_hash":"5bb1078972885f7ac1ba60fdc904e99944086bcb964e5c5d699e7eaa01abdbb8","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:30Z","homepage_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf","id":"anthropic-opus-4-8-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Table 8.1, p.194; §8.10.1, p.202","name":"Anthropic · Claude Opus 4.8 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0dc76ca4c859b3a6a92b7b6a","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28","basis":"evaluation_at","evaluated_at":"2026-05-28","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b1f62a1afeecf477c9e45535b900ff45610c80046d2d23c4a94b1e6e8ac6c460","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/anthropic-claude-opus-4-8","source_model":"Claude Opus 4.8"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"27acd100566eafe72e7379beb5a9d17cdb73daf3372f02243efdffe2b979e646","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.8,"score_display":"49.8","source_stated_rank":5},"run_fingerprint":"955f2a0c4c90c88d5f481e15788fe84d330157385d99bdc4596397749f4c926e","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7034d2f91b30bf88b9fbb3bc","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"c8bf90bc627e3d52112bde6b0dcbb67ee6eca772609ddd4255fc435825fe0a1d","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"49e3ab4763a7a0d5304180536820bc24a4acfb70961159194f84aa2f5b3ee844","result":{"confidence_high":59.921029,"confidence_low":56.054422,"sample_count":2500,"score":58.0,"score_display":"58.0","source_stated_rank":null},"run_fingerprint":"1589dd9b3df455aada923295bd39b3b732b06a1a9b0d0a2f58617ec208e24a1f","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0dc76ca4c859b3a6a92b7b6a","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"363728edc21a7d7cac775f4a6006c88d339cc4e193f1ff3f5643dbd180c7364d","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"xhigh","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"fccc1cfd9949d76f133de762dedaf6c0b853251f086c0d221d19394124608d87","result":{"confidence_high":59.524114,"confidence_low":55.652565,"sample_count":2500,"score":57.6,"score_display":"57.6","source_stated_rank":null},"run_fingerprint":"2b2563af990b2fd240547f88e1f4ec4dddebb8a33f48550d7bf14e88630918f3","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9a48a797f3c1c4753932b3c8","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"0d09208c3c8f68218d985ad430d44b1cbb0bc619c5d38c5746081ad7ad3479e9","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"high","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"70775ee4eae58805e094c269c02e142129c0bfc406e19bade9a017138f2d6468","result":{"confidence_high":57.637002,"confidence_low":53.745507,"sample_count":2500,"score":55.7,"score_display":"55.7","source_stated_rank":null},"run_fingerprint":"c1afaf028e2c0bca0c2687ac6897b600d8df701b52b922d7dcc43a78c2dc464c","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_b8404dc6a56127ee2b9a9009","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"bff954650dc0bb32b6faa513a30f0e1ee8f2787008f5f34d36954d4d8edde6d1","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"medium","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"16356ae1a8409663756f59f5ce9d7004997784fe4931d42c3d25b0e035524cbb","result":{"confidence_high":57.139914,"confidence_low":53.24413,"sample_count":2500,"score":55.2,"score_display":"55.2","source_stated_rank":null},"run_fingerprint":"670d47a25c91147da10f2a99207cd5a9207dbff29db83f75c084e3d9d46501ae","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_6e96e0354ddc6b072f329c83","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"f7eb4e75cef65a1ed5c4de2ba452a53af1ff29451322b4b83e2f063e34210f74","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"low","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"d685bd20de3886c964c5c7bbadc5f8e86f29eec0eacb141c3a416e73e220be38","result":{"confidence_high":52.158173,"confidence_low":48.241213,"sample_count":2500,"score":50.2,"score_display":"50.2","source_stated_rank":null},"run_fingerprint":"4d59345e3ef0cda0ea1749e316e4baa9934c373b27565e9161652743f916be4d","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_53fdff1a804798a16c0e9f0d","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-28","status":"stale"},"measurement_id":"aa418fdad199dc11147d6498a73f5d4342eec44b7b7480f1dd0b814d9c604d5e","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Table 8.1, p.194; §8.10.1, p.202"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"vendor-reported HLE grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-05-28","source_content_hash_method":"raw_bytes","source_content_sha256":"0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f","source_published_at":"2026-05-28","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"3f8383df7356e462639e6353cd66465d9b34b820914c238a53f7bb19e5368b88","result":{"confidence_high":59.821813,"confidence_low":55.953946,"sample_count":2500,"score":57.9,"score_display":"57.9","source_stated_rank":null},"run_fingerprint":"4d811d26879002c9ab9d0ff5438a0651e311ad946fd9e2c13c3be5ba62c51d70","source":{"content_hash":"5bb1078972885f7ac1ba60fdc904e99944086bcb964e5c5d699e7eaa01abdbb8","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:30Z","homepage_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf","id":"anthropic-opus-4-8-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Table 8.1, p.194; §8.10.1, p.202","name":"Anthropic · Claude Opus 4.8 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0dc76ca4c859b3a6a92b7b6a","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:27:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:27:24.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7e441e112c3814207ff64ebf0a862b236ff1d0f02680ee310c03d4690c7bbb3d","metric_details":{"best_score_across_scorers":"0.53","model_release_date":"2026-05-28","stderr":1.5790799515836722},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ViC6pxQZUVS3E9M9iEubF7","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FViC6pxQZUVS3E9M9iEubF7.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ViC6pxQZUVS3E9M9iEubF7.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"8048e6f0aa21e53a5260b9c83d5b08a693fa97069d9c4dc25a37482175a1a6c8","result":{"confidence_high":56.094996705103995,"confidence_low":49.905003294896005,"sample_count":1000,"score":53.0,"score_display":"53.0","source_stated_rank":20},"run_fingerprint":"8490b1baccdd65d6799eb2afe4f9b48c946ac406f74b3147d838e46f3aa04bb4","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":126,"at":"2026-06-01T14:18:01.761133Z","basis":"evaluation_at","evaluated_at":"2026-06-01T14:18:01.761133Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c8c2afbbb314ade81d5b0e4850a935f779600db26d68e2b1f68b2f54c3cb764d","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-opus-4-8-default","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"28c80ec3ef9c45ba0b0f3cfe915f605e5f0aeef68868621e60098f118dba5b2c","result":{"confidence_high":47.68375062678194,"confidence_low":41.38398914518318,"sample_count":null,"score":44.53386988598256,"score_display":"44.5","source_stated_rank":15},"run_fingerprint":"7b80473c8ad2f7dd0b34b0348038b998af9e0258a482f2d70e7f7068f97e154c","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ccc88b2fd06fbf7541d5ced2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6697ccb35ae9b887125236b9273b38793663ff7ffd656777032b49e8bfb33fb1","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.339261,"date_is_proxy":true,"latency_seconds":133.876,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.984},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","few_shot_accuracy":91.919,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-8","zero_shot_accuracy":92.929},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"e98b76f19e897abe696d37c5797d8bbe8dc4e0954d88053b9640048363f3d164","result":{"confidence_high":96.31264,"confidence_low":88.53536000000001,"sample_count":396,"score":92.424,"score_display":"92.42","source_stated_rank":16},"run_fingerprint":"05f4c47c416ee05300fe99a8ec2f43eebfb6c26c7ff3ef8e5b3a556fb3b820ca","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":119,"at":"2026-06-07T23:03:11.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-07T23:03:11.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"6126c096c157c41af8bbd6f5cbbad9a682c34254bc5a37e3fefbae9bdd9af0bd","metric_details":{"best_score_across_scorers":"0.9103535353535354","model_release_date":"2026-05-28","stderr":1.9157125794612027},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dqgqP2uEPswfECCVj2Kbr9","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e2afdd10de1a9a5e71a959a0fba16b759a09edd09b23ddf6b0bff857f506bc41","result":{"confidence_high":94.7901501910975,"confidence_low":87.28055687960958,"sample_count":null,"score":91.03535353535354,"score_display":"91.0","source_stated_rank":32},"run_fingerprint":"4e7f973650c9182d34885b33d3027be56bf7f431a2319fd74d79ea5623ad1054","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:19:23.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:19:23.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9144792b15c85aec54195be60c08d3da845a6abb8f6ff05c406261a3bba8adb5","metric_details":{"best_score_across_scorers":"0.8838383838383839","model_release_date":"2026-05-28","stderr":2.2828881775249354},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"GThtce6GAmRAS8srQyGQoA","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGThtce6GAmRAS8srQyGQoA.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/GThtce6GAmRAS8srQyGQoA.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"807dce4480c9d77eb0725830ef6e60f90eec6783de72ffb658580253189b7987","result":{"confidence_high":92.85829921178725,"confidence_low":83.9093775558895,"sample_count":null,"score":88.38383838383838,"score_display":"88.4","source_stated_rank":58},"run_fingerprint":"8f7fe9cb4638a204391388255c0860a9c1cae95419ae76a627ef4cb241d29bbb","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_889e221c2cf191b39b603fd5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:19:27.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:19:27.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f79cd51d85d555a6721a97620247496c492cf51ed823c44095fb4204181da308","metric_details":{"best_score_across_scorers":"0.8535353535353535","model_release_date":"2026-05-28","stderr":2.5190921114603886},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ckbZdkrXPwbFdjn6bmUQsi","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FckbZdkrXPwbFdjn6bmUQsi.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ckbZdkrXPwbFdjn6bmUQsi.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"336506a94ed340cc8657624166f7d6e9d96e54aa5d3cc67a6af325b149bf7f92","result":{"confidence_high":90.29095589199771,"confidence_low":80.41611481507299,"sample_count":null,"score":85.35353535353535,"score_display":"85.4","source_stated_rank":87},"run_fingerprint":"2f0bf71fc297808897c1a775edb8810c581348c85aef45e42ac97741d103492a","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_50e3edadf9217a1e309de819","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":98,"at":"2026-06-28T14:53:09.916993Z","basis":"evaluation_at","evaluated_at":"2026-06-28T14:53:09.916993Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fc59beefafd4d9d4d427c3c4552e2df7d0e2e9678d35e478194ba60ea9586ffe","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-opus-4-8-default","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"24dda5524931d430c3f5435ee2a361776ba96905d7c39c5267fec301810e84ca","result":{"confidence_high":88.46341075719742,"confidence_low":78.0340511717366,"sample_count":null,"score":83.24873096446701,"score_display":"83.2","source_stated_rank":16},"run_fingerprint":"59fc586e36d0e2945084a3404d8f23c7728e5550c62b0ba5019e4a5881c99a39","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_501d9876a1e201d4894bb96e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c0c114548c37731da96102a55f5a89e09985ae73ca4b873afee6a278afba0077","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.060716,"date_is_proxy":true,"latency_seconds":23.44,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.303},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-8"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"66663e630d96d178849d33be72aaebf428ca1a954ce6e09740eb95b77a809cc2","result":{"confidence_high":90.17887999999999,"confidence_low":88.99112,"sample_count":null,"score":89.585,"score_display":"89.6","source_stated_rank":9},"run_fingerprint":"fcb366033e63bd00a223dffee5a765cd11c1abc097c4c503718958286b5805f3","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":119,"at":"2026-06-07T23:44:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-07T23:44:54.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a70cd319624d62dcfb5ff90ed909ddf6991fbb45fd0d1ba8ac22ee4c17e7e926","metric_details":{"best_score_across_scorers":"0.9833333333333333","model_release_date":"2026-05-28","stderr":1.4101901870444153},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"37pJqPpXTE6ia9ikmmGPJ4","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3363252c17a2fc64205757514a0be605af0eedfdaae23c490b34e31ca4327790","result":{"confidence_high":100.0,"confidence_low":95.56936056672627,"sample_count":45,"score":98.33333333333333,"score_display":"98.3","source_stated_rank":21},"run_fingerprint":"9a7a5d54c1cedeb9dd9883ca45545dc6dcfdc11fd68a72a859ef047a6a8fda96","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:20:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:20:24.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e038caea3fc3a05e10f56ca47f5bf54e46ef3a40b459d4f0c075cbe7209df0ed","metric_details":{"best_score_across_scorers":"0.9777777777777777","model_release_date":"2026-05-28","stderr":2.2222222222222223},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"MEhV5sh6nbM8cKJGQ6HzcS","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMEhV5sh6nbM8cKJGQ6HzcS.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/MEhV5sh6nbM8cKJGQ6HzcS.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1f8ae9a0bfc9f09e945766a9eb0a355e344a1bc2148a14cae8db90aeaf8fa514","result":{"confidence_high":100.0,"confidence_low":93.42222222222222,"sample_count":45,"score":97.77777777777777,"score_display":"97.8","source_stated_rank":25},"run_fingerprint":"ee6dc72f0d3bde23a49e683167634bd9b33ddf9b58abc904c4baaa12419b35c6","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_889e221c2cf191b39b603fd5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:21:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:06.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1d70dda5c37ed52115eca5c59bfa6a8cc4a4bcf417583fdd7965ec131d433013","metric_details":{"best_score_across_scorers":"0.8444444444444444","model_release_date":"2026-05-28","stderr":5.463890236888291},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"adGE8rXdNuUFs96ioppbc5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FadGE8rXdNuUFs96ioppbc5.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/adGE8rXdNuUFs96ioppbc5.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"02c6d52da8c753fc35f0f8207c5c671e76f7db4227985a8547d64f2a958ef141","result":{"confidence_high":95.15366930874549,"confidence_low":73.7352195801434,"sample_count":45,"score":84.44444444444444,"score_display":"84.4","source_stated_rank":93},"run_fingerprint":"fa31b30d504e33b555c53b243b70886c66074656ef35938991392f4bc047b0dd","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_50e3edadf9217a1e309de819","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T15:25:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T15:25:15.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c3d6c4433b0fce4dc6f8752c5f291eb495ce70530aa4913ddc90d284d57911db","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.8","leaderboard_private_problem_count":285,"model_release_date":"2026-05-28","public_example_count":10,"stderr":2.3735633163877066},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"GVmgtgKMwEqc54f3jA7auP","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGVmgtgKMwEqc54f3jA7auP.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/GVmgtgKMwEqc54f3jA7auP.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 4.8","thinking":null,"upstream_model_id":"claude-opus-4-8_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"dd340eb941ab0c21506a3592392589d334adab0312a07863d24010721c5eb092","result":{"confidence_high":84.65218410011991,"confidence_low":75.34781589988009,"sample_count":285,"score":80.0,"score_display":"80.0","source_stated_rank":15},"run_fingerprint":"b763177a855b14c9ae7b6b591f03470ff78c268299bdf7db7a1b7509ae81d810","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"00527c4803bbf791ae608371f8a82c7832be27cb5f2ccbf0cc9046d1777b40e2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":2.7425,"model_release_date":"2026-06-01T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-opus-4-8","model_type":"CoT","upstream_model_id":"anthropic-opus-4-8-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"785a72df191545092d039dd57501ca156b8dd8150479806c6b50c0373c9796f5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.08,"score_display":"72","source_stated_rank":50},"run_fingerprint":"c4a80e8525b6572e2b95980d8827cb4d7f17169132bd51cb08e7246683f103a9","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7787e4da5cc358912521b6362e5e04ee9bc218498b23d8619b6404a42de8d930","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":2.3896,"model_release_date":"2026-06-01T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-opus-4-8","model_type":"CoT","upstream_model_id":"anthropic-opus-4-8-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1124a160a5f6e43d4800168d3e4aaae7bc05b87b2081ab6611e3385006518f9d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":71.67,"score_display":"72","source_stated_rank":52},"run_fingerprint":"309c8e8a8c3598218e15a2ac1fb2f01db782322c816df8ce31bb15032380d081","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_16c9495bd6a270687b586303","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T13:33:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:02Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c161b09ba815592cfd4ab6c40e8514e74a482e27d55089fb12b59dc63d8da837","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.6818,"model_release_date":"2026-06-01T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"anthropic-opus-4-8","model_type":"CoT","upstream_model_id":"anthropic-opus-4-8-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bc7e929d8d9b3189b281cda87cdc63c9d23c246bae017833c09b49f8d71e65d1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.22,"score_display":"62","source_stated_rank":70},"run_fingerprint":"e518cd7a40f7da9e09d5baf9c0d9fd7da4800a4726160ab5bbd40898912db2e5","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_889e221c2cf191b39b603fd5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"56eab9eef83b6ec72e10244b0a174e9d00a03de1eff7da6d88ffba47bb8d1cd0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"2.7425","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (High)","source_effort":null,"source_model_version":"claude-opus-4-8_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"2c981652fc46681f23f7ba17331b7b19751013d1b7f0617f10cf30cdf2d2685a","result":{"confidence_high":76.46238,"confidence_low":67.23136,"sample_count":360,"score":72.08,"score_display":"72.1","source_stated_rank":53},"run_fingerprint":"20e3db8e3db5ed70570c6f628ef22bac5ff7cd3965610b11a3f4b012b4ccaf4c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b296d233adce6ebb2690b3c668dd6eb8ac8713963b705c4829cea2574c61b6e1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"2.3896","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (Medium)","source_effort":null,"source_model_version":"claude-opus-4-8_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5850e2316472f4a1be8edac5af95d6515932aa210b5a905d524f61acfad1e152","result":{"confidence_high":76.076964,"confidence_low":66.805433,"sample_count":360,"score":71.67,"score_display":"71.7","source_stated_rank":55},"run_fingerprint":"360a2db0505707d5b08f28e2f791ae4f45a4f5a9ca04ce3f13292c52b4c40c2b","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_16c9495bd6a270687b586303","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7a1c5167668d98e6084ec94d40a02cde5ee92a5d6d6e96596a88fb5b62277f1d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.6818","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (Low)","source_effort":null,"source_model_version":"claude-opus-4-8_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d24794a40550403b79dce26386b75c141acf1ec7a5e7aabfb6ae8d492c347b2c","result":{"confidence_high":67.074556,"confidence_low":57.107395,"sample_count":360,"score":62.22,"score_display":"62.2","source_stated_rank":73},"run_fingerprint":"75e770d43df5fb97a2928249fd7410c82a86770cf1ac0638abe581c8dd2df493","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_889e221c2cf191b39b603fd5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T15:25:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T15:25:15.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b52157216674e1ac5b19d84403656d9e0428535521a864b738e89458e480a3c2","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.5609756097560976","leaderboard_private_problem_count":41,"model_release_date":"2026-05-28","public_example_count":2,"stderr":7.846686801046543},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"JiqgqBTfkGx797uowZgJbk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJiqgqBTfkGx797uowZgJbk.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/JiqgqBTfkGx797uowZgJbk.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 4.8","thinking":null,"upstream_model_id":"claude-opus-4-8_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"75931d8996e8e66e261aebde906fd9ff2c0c295340bc18b20aa6eb2f379cba06","result":{"confidence_high":71.47706710566098,"confidence_low":40.718054845558534,"sample_count":41,"score":56.09756097560976,"score_display":"56.1","source_stated_rank":20},"run_fingerprint":"78122b184984438134a33a3d6044a1df9426930fcfebf69633b396ec87f6fb84","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c62b71bb39aa15b28ed2d7be1db4030713815ac7cbcad330c9629b6f90faaf67","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.120663,"date_is_proxy":true,"latency_seconds":49.044,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.819},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-8"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"637ec0a3adf7ab1ea50053fd8090e240090ea8193dfda915253be371b96962c5","result":{"confidence_high":88.19524,"confidence_low":84.98476000000001,"sample_count":null,"score":86.59,"score_display":"86.6","source_stated_rank":18},"run_fingerprint":"b8d2672a4952266da8d841939eea149a10ee2a6354fee71f6d2d90281927571d","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"19cec0f080a17443ffdbe5a0874c0878e82a11d1f556268ef4224f8d8d3fb3aa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-opus-4-8_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"eae1f751d22a1666394481c64aad373db1009d5cf59454f549cbd63424bacc61","result":{"confidence_high":31.678669,"confidence_low":13.027407,"sample_count":71,"score":20.8571428571429,"score_display":"20.9","source_stated_rank":48},"run_fingerprint":"f1b2a70e682be4ec23f05da5e53a901904f5ab7f143218b121043395c7737e77","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7ef626312f32b28737cc3eb0338d56d4d3ed83e8f852a602c4d8a345760503dc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-8_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"b04a4db496b6f73099c0c613de0dc31bbeac6138402dfe202360cdaa95e7907f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.8,"score_display":"64.8","source_stated_rank":17},"run_fingerprint":"efcdab5ddc380e0419b43524449fbd420363e1d09ef5b24981c592e1a4301f0f","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c95142f46157a2f984bb1f9f","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":128,"at":"2026-05-29T17:00:51.000Z","basis":"evaluation_started_at","evaluated_at":"2026-05-29T17:00:51.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5312ba00e27fbdfe8da9420205b41d514c5aff7dba2e1ccb570d728dd4e9434d","metric_details":{"best_score_across_scorers":"0.34","model_release_date":"2026-05-28","stderr":4.760952285695233},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bwak5Cq3LBthfoboicGZUo","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"417d9f05368f51e9ff40d18634c559932f8decd6b85cc349bd3739a2390cfcca","result":{"confidence_high":43.33146647996266,"confidence_low":24.668533520037343,"sample_count":100,"score":34.0,"score_display":"34.0","source_stated_rank":43},"run_fingerprint":"14e79f0eb577908434a927768ae03ac607320919511c137f31361bd2a38fe331","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:19:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:19:39.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5fdb3f462ce2f2443b195b00a0a619129a5333465830f122189aad5be0fc9458","metric_details":{"best_score_across_scorers":"0.29","model_release_date":"2026-05-28","stderr":4.560480215720686},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"5u3LkMVq4Dy8CfiwA2EAVr","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5u3LkMVq4Dy8CfiwA2EAVr.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/5u3LkMVq4Dy8CfiwA2EAVr.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f2709e3edae2f89e182adf829a199a543abe38b44c87efde5c13b7d7ea6d22e0","result":{"confidence_high":37.93854122281254,"confidence_low":20.061458777187454,"sample_count":100,"score":28.999999999999996,"score_display":"29.0","source_stated_rank":54},"run_fingerprint":"06a7350548ad6668b2b180681e709698df864d3332ee766b0be35825b57b4823","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_889e221c2cf191b39b603fd5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:20:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:20:20.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e8c82b12aa95774b458759803687abea5c500305d8bcba66ff0750fec12e5335","metric_details":{"best_score_across_scorers":"0.13","model_release_date":"2026-05-28","stderr":3.379976689896311},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"oU38rqq2TvJTxvvPsnEQsH","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoU38rqq2TvJTxvvPsnEQsH.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/oU38rqq2TvJTxvvPsnEQsH.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"873c7657ede44353a66cb5e2714e0864606a5e1a76ba80544407e180b8b229a9","result":{"confidence_high":19.62475431219677,"confidence_low":6.375245687803231,"sample_count":100,"score":13.0,"score_display":"13.0","source_stated_rank":120},"run_fingerprint":"2e5aad670c21f51f21fb4aa96c0332a70351d655fcb708be3634c80d38211817","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_50e3edadf9217a1e309de819","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b7f2d546fd8f7c2d87967af7eed0dca813dc371bb56b27da45f7b31ff0c581fd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.332","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (Max)","source_effort":null,"source_model_version":"claude-opus-4-8_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"9f87e4176f2a64b2f0804476be952e926e8821a4e85daebece5d9588a07bbac2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.5,"score_display":"92.5","source_stated_rank":54},"run_fingerprint":"9402d20923f00a977443edc9385e375d91f05e233bb351d8199c9a18397dcabe","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fa3aa8fa0a2bb5be752aee1418835bdb30321f02488fa7ad018f5681227fac01","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.0416","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (High)","source_effort":null,"source_model_version":"claude-opus-4-8_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"bea61cd5fa5b646508379cf0bb48e5a24e2af55a718a045cd5258846f1979dbf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.0,"score_display":"92.0","source_stated_rank":59},"run_fingerprint":"6c52b286bb0573e3c073766fb23b640ac7e07645bf68296c97975c32754d03ce","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1425db75b73c9a4770c911486665468701cd6a4df1a4c59b8b9e272e295fc322","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.9115","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (Medium)","source_effort":null,"source_model_version":"claude-opus-4-8_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6588b0be38fa7bf2c8d801ace0c70cf38df2f83b89673d26a6d23a859bac5ff2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.5,"score_display":"91.5","source_stated_rank":64},"run_fingerprint":"c088fc16fcb0745b5b81681b1a2b5fba43cf89138e111ae684131b798c8aa5ee","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_16c9495bd6a270687b586303","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"150e12bc423fc5ee04f7a906f17030b4d2b087f213f3e56ad651ef35affc49f9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.6711","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (Low)","source_effort":null,"source_model_version":"claude-opus-4-8_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c0c90d19d0af816fa978b02f3058dc299852a30f65e3afa778b12cd3f3237a35","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.0,"score_display":"88.0","source_stated_rank":80},"run_fingerprint":"6d5e3e1f861654488b7fc6e7a5b4ec31c75e8a7de8ff7567e9ba2601789e72e6","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_889e221c2cf191b39b603fd5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":130,"at":"2026-05-28","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-28","status":"stale"},"measurement_id":"2e3c572d5dd78cbc82ffe98c88e0576e77b4cd0139394120a0193f4b8fbd4ad6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run; vendor scores using a different agent or tool setup remain separate supplemental runs.","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","cost_per_test_usd":1.923224,"date_is_proxy":false,"latency_seconds":566.945,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-opus-4-8","source_locator":"Results · Overall accuracy","stderr":1.423},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"66f445879d9c7eefc688961e380b7f95cd09c6cbe7c8dc2a5a2ed7fdf3813aed","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-8"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"133dff3c348980f17ccca03da3f3d1c7d8e27e4431cb4eba60929fc9480a32b7","result":{"confidence_high":91.38907999999999,"confidence_low":85.81092,"sample_count":500,"score":88.6,"score_display":"88.6","source_stated_rank":12},"run_fingerprint":"0194c8d27620bd0206841cca334a73c522cde7df58b0e6c35009276db8d4a5a2","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ed49ef6cb40ad079db5a0835","split_or_window":"verified-500","unit":"percent","version":"Anthropic 500-task evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-28","status":"stale"},"measurement_id":"cbc07f569eb3f7143cde51046cc7e78d82215c236d67f819c129b8d91abc99f4","metric_details":{"comparison_warning":"Vendor run over 500 tasks; do not equate it with Epoch runs over 484 tasks.","comparison_warning_code":"swe-vendor-500-vs-epoch-484","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.2, pp.195–196"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"swe-vendor-500-vs-epoch-484","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; five-trial average","question_count":500,"reasoning_effort":"max","result_published_at":"2026-05-28","source_content_hash_method":"raw_bytes","source_content_sha256":"0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f","source_published_at":"2026-05-28","tools":"not reported"},"run_count":5,"scaffold":"unspecified agentic harness","tools_allowed":"not reported"},"protocol_fingerprint":"f595922acd4f9772f570b876988ce1fa827612d9ebb2792b02bdbe22633c821e","result":{"confidence_high":91.096351,"confidence_low":85.515029,"sample_count":500,"score":88.6,"score_display":"88.6","source_stated_rank":null},"run_fingerprint":"7044fc5eba1587c26bcc502552c0c09fc67d2ffe9d4a806f4865efb4ac9f2e1c","source":{"content_hash":"5bb1078972885f7ac1ba60fdc904e99944086bcb964e5c5d699e7eaa01abdbb8","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:30Z","homepage_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf","id":"anthropic-opus-4-8-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.2, pp.195–196","name":"Anthropic · Claude Opus 4.8 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0dc76ca4c859b3a6a92b7b6a","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"dd2c952d23d70dae7e0d09e246c2bf130c357575d42d7ed66a994a59323c60f1","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-05-28","pass_at_5":67.27272727272727,"potential_contamination":true,"sem":1.199173268933903},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","potential_contamination"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"Claude Opus 4.8-xhigh__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"7645eeaa8bc0d9f452dcbd478ae9fc099df4ca50dd375c7cc8a3963a612ee618","result":{"confidence_high":58.895834152565,"confidence_low":54.195074938344106,"sample_count":null,"score":56.545454545454554,"score_display":"56.55","source_stated_rank":null},"run_fingerprint":"365f38b1eba285d4751b16405bf0980756ac038a22f444a4db94202c26c59ab4","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_576644939af96eac30f5bf8d","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"1d530abfe9a116a88ad444aef13b9e074953d117c33e68d3e1cf2e7d2e1925dc","metric_details":{"ci_attempted":447,"ci_half_points":3.7134306280944758,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":243,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":94.64429530201342,"mean_cache_tokens":9693014.304250559,"mean_cost_usd":8.006355704138702,"mean_duration_seconds":2198.774075261745,"mean_input_tokens":9854652.170022372,"mean_output_tokens":86088.7874720358,"median_agent_steps":90.0,"median_cost_usd":7.049357750000001,"median_duration_seconds":1829.190182,"median_input_tokens":8117323.0,"median_output_tokens":77961.0,"median_output_tokens_to_pass":76018.0,"median_peak_context_tokens":153870.0,"n_attempted":447,"n_passed":243,"n_tasks_attempted":113,"n_tasks_passed_any":91,"pass_at_4_points":80.53097345132744,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_4_8_xhigh","source_model_version":"claude-opus-4-8","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"4ad987fa0c4aefb8ce2c11684129ff04829491fd3bdfed633b66139aa56c3c1d","result":{"confidence_high":58.07584673547702,"confidence_low":50.64898547928808,"sample_count":447,"score":54.36241610738255,"score_display":"54.4","source_stated_rank":40},"run_fingerprint":"71a0f61a00fa655e502a3f9271ff3bd4787f6ad87f5eac0047c2db9393e1ecbe","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_576644939af96eac30f5bf8d","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"9b310b8031d0bbaffbe57eaaab45384892e5e882ef0ed2c1d90d5c676bfc2b1c","metric_details":{"ci_attempted":452,"ci_half_points":4.561609145755844,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":234,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":72.5,"mean_cache_tokens":4807501.1283185845,"mean_cost_usd":4.281940017699115,"mean_duration_seconds":1066.903783732301,"mean_input_tokens":4908687.0265486725,"mean_output_tokens":50063.52654867257,"median_agent_steps":67.0,"median_cost_usd":3.6635345,"median_duration_seconds":887.7987455,"median_input_tokens":3814284.0,"median_output_tokens":44476.0,"median_output_tokens_to_pass":43664.0,"median_peak_context_tokens":94158.5,"n_attempted":452,"n_passed":234,"n_tasks_attempted":113,"n_tasks_passed_any":88,"pass_at_4_points":77.87610619469027,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_4_8_high","source_model_version":"claude-opus-4-8","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"5587ee174042be611430cdf389b61a77aa1b2b330c91bd159b063d7bb30b9691","result":{"confidence_high":56.33152065018062,"confidence_low":47.20830235866894,"sample_count":452,"score":51.76991150442478,"score_display":"51.8","source_stated_rank":48},"run_fingerprint":"49528ddc633249cc22b72e0e4211617d5586d62ffbec5d30793c4738c14c7d1a","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"01e492beba8bd22180b6961a5a7abfa2a26e46327533ba6b5b4828b9804f9902","metric_details":{"ci_attempted":452,"ci_half_points":2.2392058617374517,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":220,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":65.57300884955752,"mean_cache_tokens":3757520.9115044246,"mean_cost_usd":3.4438571028761062,"mean_duration_seconds":880.0083837986726,"mean_input_tokens":3843602.6681415928,"mean_output_tokens":41313.45132743363,"median_agent_steps":60.0,"median_cost_usd":2.84043425,"median_duration_seconds":729.1386,"median_input_tokens":2851276.5,"median_output_tokens":36647.5,"median_output_tokens_to_pass":36116.5,"median_peak_context_tokens":79616.0,"n_attempted":452,"n_passed":220,"n_tasks_attempted":113,"n_tasks_passed_any":86,"pass_at_4_points":76.10619469026548,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_4_8_medium","source_model_version":"claude-opus-4-8","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"c57f414ce45a0138f17388d23d35d5c940b3d95d08496e4f48f52c257fbb828c","result":{"confidence_high":50.91177223341886,"confidence_low":46.43336050994396,"sample_count":452,"score":48.67256637168141,"score_display":"48.7","source_stated_rank":51},"run_fingerprint":"e5d160ba64570cca74f2d24fb184db69e03e68c8393dd6a8d0196fb5f4600e14","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_16c9495bd6a270687b586303","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"e62f1beb960d63c46d7034ac89edf19dc5713c3d8606964f9667bc9af1bf6cf8","metric_details":{"ci_attempted":451,"ci_half_points":1.4635847585691517,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":184,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":53.97782705099778,"mean_cache_tokens":2354415.691796009,"mean_cost_usd":2.293369216740577,"mean_duration_seconds":651.8271712993347,"mean_input_tokens":2417987.955654102,"mean_output_tokens":28922.736141906873,"median_agent_steps":47.0,"median_cost_usd":1.8240152500000004,"median_duration_seconds":517.146142,"median_input_tokens":1658113.0,"median_output_tokens":25291.0,"median_output_tokens_to_pass":24500.0,"median_peak_context_tokens":57903.0,"n_attempted":451,"n_passed":184,"n_tasks_attempted":113,"n_tasks_passed_any":77,"pass_at_4_points":68.14159292035397,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_4_8_low","source_model_version":"claude-opus-4-8","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"aa6d8d25f289d9a1c873a34e61b0a15c5ddec85963cb05f31a53bd838a623ad1","result":{"confidence_high":42.261810922648976,"confidence_low":39.334641405510666,"sample_count":451,"score":40.79822616407982,"score_display":"40.8","source_stated_rank":58},"run_fingerprint":"6786177782809cd4371ddd8341f6db83bd81e7f77a2e067e96be9cbf798f872e","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_889e221c2cf191b39b603fd5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_a9dd3959b842003dcf9c857d","split_or_window":"111-of-113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"285b09f3f3cfca7357e57e8a5adc98c50715e45abeece596d2d9cecb2108ddc4","metric_details":{"ci_attempted":429,"ci_half_points":1.7648153474292538,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":253,"comparison_warning":"DeepSWE reports this run over only part of the 113-task v1.1 set. It is kept as supplemental evidence and cannot replace a complete comparable run. generated_at dates only the leaderboard snapshot.","comparison_warning_code":"deepswe-owner-partial-task-set-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":120.0,"mean_cache_tokens":16817766.494172495,"mean_cost_usd":13.222583593240094,"mean_duration_seconds":3492.740812641026,"mean_input_tokens":17047938.554778554,"mean_output_tokens":135031.67132867133,"median_agent_steps":116.0,"median_cost_usd":12.353505999999996,"median_duration_seconds":2883.384473,"median_input_tokens":15368311.0,"median_output_tokens":129180.0,"median_output_tokens_to_pass":125257.0,"median_peak_context_tokens":225741.0,"n_attempted":429,"n_passed":253,"n_tasks_attempted":111,"n_tasks_passed_any":88,"pass_at_4_points":79.27927927927928,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-partial-task-set","comparison_warning_code":"deepswe-owner-partial-task-set-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_opus_4_8_max","source_model_version":"claude-opus-4-8","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"partial","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"b1470011f6ed0bd72b91d21d12a8574d303fcc091bf8991c62b8fdbec7de5f2a","result":{"confidence_high":60.73917432178823,"confidence_low":57.20954362692973,"sample_count":429,"score":58.97435897435898,"score_display":"59.0","source_stated_rank":35},"run_fingerprint":"f4d10282e4264f62ab5fddbbbe7bf67f403fe31e9df20693ad73acc86fcc5194","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-partial-task-set","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"40b19f97ab54a87f3064815ebf28ef09e4e79e0f882d16abf64b5726d66c38bc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"120.0","mean_cost_usd":"13.222583593240094","mean_output_tokens":"135031.67132867133","model_release_date":"2026-05-28","notes":null,"pass_4":"0.7927927927927928","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-4-8 (max)","source_effort":"max","source_model_version":"claude-opus-4-8_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"7dc1224485b67c72032ca1be5b87318481508bb84ad0ae7ba08db7ef9d310614","result":{"confidence_high":60.739174321788234,"confidence_low":57.20954362692972,"sample_count":113,"score":58.97435897435898,"score_display":"59.0","source_stated_rank":34},"run_fingerprint":"7df4c65795382a0e70871c82800984573fb875729444d27314456b4b6b9bac9e","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"86724ba4592ee6007633959fedf42b99be206c68e5184fe7039712d344bd66de","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"94.64429530201342","mean_cost_usd":"8.006355704138702","mean_output_tokens":"86088.7874720358","model_release_date":"2026-05-28","notes":null,"pass_4":"0.8053097345132745","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-4-8 (xhigh)","source_effort":"xhigh","source_model_version":"claude-opus-4-8_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"d07f78788e39d492b341c4dd7d4c973564a8ab1676c4f8d3e7a37d524eee7377","result":{"confidence_high":58.07584673547703,"confidence_low":50.64898547928807,"sample_count":113,"score":54.36241610738255,"score_display":"54.4","source_stated_rank":39},"run_fingerprint":"9528f528ea5d3881d0eed4f940e93129377bf89624ee094c28618423659cd7cb","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_576644939af96eac30f5bf8d","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6d9e7e737092b9fbba32f85997102a84a29bf059ef41cc46d7277638bd26d394","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"72.5","mean_cost_usd":"4.281940017699115","mean_output_tokens":"50063.52654867257","model_release_date":"2026-05-28","notes":null,"pass_4":"0.7787610619469026","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-4-8 (high)","source_effort":"high","source_model_version":"claude-opus-4-8_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"8c014457cdbece0cffb6b896782eb613e45b79d89c1fb4dc6eb68d80f64099f4","result":{"confidence_high":56.33152065018063,"confidence_low":47.20830235866894,"sample_count":113,"score":51.76991150442478,"score_display":"51.8","source_stated_rank":46},"run_fingerprint":"16015fc7cb181514d13318478a59cb073962612b1a5212990817a201ead047b9","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0077fcf3750736cb7d439ccf2f0c6f52465e14cb3078e4163d3cec748a49e7b8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"65.57300884955752","mean_cost_usd":"3.4438571028761062","mean_output_tokens":"41313.45132743363","model_release_date":"2026-05-28","notes":null,"pass_4":"0.7610619469026548","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-4-8 (medium)","source_effort":"medium","source_model_version":"claude-opus-4-8_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"d3dacc710fb39ea1a67d6aab37255ac5f6bb1421405fce19635176939fc66af2","result":{"confidence_high":50.911772233418866,"confidence_low":46.43336050994396,"sample_count":113,"score":48.67256637168141,"score_display":"48.7","source_stated_rank":49},"run_fingerprint":"e819f50e68f48c6ee49403350af717d118b66cc1f143224a263027a817c52028","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_16c9495bd6a270687b586303","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"eea73496dbcc83132521e53c62138cc60032c71e25710f8ecbf327d5d3c0133c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"53.97782705099778","mean_cost_usd":"2.293369216740577","mean_output_tokens":"28922.736141906873","model_release_date":"2026-05-28","notes":null,"pass_4":"0.6814159292035398","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-opus-4-8 (low)","source_effort":"low","source_model_version":"claude-opus-4-8_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"9551061b3aaf9e478d129a81d4069177d80f67167aea991a5f226cbbfa2d4cc0","result":{"confidence_high":42.261810922648976,"confidence_low":39.334641405510666,"sample_count":113,"score":40.79822616407982,"score_display":"40.8","source_stated_rank":56},"run_fingerprint":"9875e7edf6b56bd1c1649465da7b5a5bba6a46cb6e6fd67b45bc6fe82ac5dde8","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_889e221c2cf191b39b603fd5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_e361abe5266a3ddfdcef017b","split_or_window":"openai-provider-comparison","unit":"percent","version":"DeepSWE v1.1 · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"36f7138d79daed0a9f43cdff585cfc79d09b310f3f8d81e697ab74fe219a3586","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · DeepSWE v1.1"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value for DeepSWE v1.1; competitor rows are secondary re-reports; exact scaffold, task coverage and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-05-28","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"coding-agent tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"c3c691f7cbafb741d93e9565dbadcfd737f921bd5f66b34135002c1b1009768d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.0,"score_display":"59.0","source_stated_rank":null},"run_fingerprint":"cf9c394a76c899ad2375ed8e9c9e77c426d552d452f9f08704e267bfaa6f5d1e","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · DeepSWE v1.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_300bc637ebe4d241305bff6b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_6323abbd504101af2cdaac58","split_or_window":"vendor-public-own-harness","unit":"percent","version":"SWE-bench Pro · Anthropic provider run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-28","status":"stale"},"measurement_id":"f03bb310ef0f5a846c3158b41c4124a97dd2301dc7518b625c9a59d2c0213d5f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.2, pp.195–196"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; default sampling; five-trial average; exact public-task revision not restated","reasoning_effort":"max","result_published_at":"2026-05-28","source_content_hash_method":"raw_bytes","source_content_sha256":"0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f","source_published_at":"2026-05-28","tools":"software-repository shell and code-editing tools"},"run_count":5,"scaffold":"Anthropic agentic harness","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"d76277efd0e4378d0e582db8da54e94f2a8b7f7e587069b2a9b76d9caa3a10c0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.2,"score_display":"69.2","source_stated_rank":null},"run_fingerprint":"ed222bfa8cd39d0a1f992a18eb509a8edced086d0e2771d9491898068a386797","source":{"content_hash":"5bb1078972885f7ac1ba60fdc904e99944086bcb964e5c5d699e7eaa01abdbb8","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:30Z","homepage_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf","id":"anthropic-opus-4-8-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.2, pp.195–196","name":"Anthropic · Claude Opus 4.8 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0dc76ca4c859b3a6a92b7b6a","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_7664765571d731e0f6ff7adb","split_or_window":"openai-provider-comparison","unit":"percent","version":"SWE-bench Pro · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"e6a8a94fd502acc5a2e150905a82abe9ea42744ebaf4f747772e8670e8057a64","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · SWE-Bench Pro"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; OpenAI rows are provider results, while competitor rows are explicitly retained as secondary re-reports; exact scaffold, task revision and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-05-28","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"deb1e7bf99d7a9b74dc9a12bd8e0058abd4ccecd6093e3f8ae550379a3ef4004","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.2,"score_display":"69.2","source_stated_rank":null},"run_fingerprint":"f64de87613ce5918ee8b80f28e71eeb3bf75f9c71117ab4602a0d791fa176479","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · SWE-Bench Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_300bc637ebe4d241305bff6b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"b283087d5832d180924438ee6458c51ac11fc02b9ce1ec32b6708962e7fc175a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"5.77","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-05-28","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"44","tokens_per_task":"71411","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.8 Max","source_effort":"Max","source_model_version":"claude-opus-4-8_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"2d5258c24207792f200dc07cb991c6780330cb8fffe9a9bc2a25c7274a0722cd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.3,"score_display":"62.3","source_stated_rank":25},"run_fingerprint":"110aabab5a4a3adb4f9a28420ec9e0b6a0f119eb00e8ef13765119bf38b4f26c","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"45a7e32fc64f43ce887cc279e83ab669f49c9c62508eddc2a811f91a84b25938","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.5","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-05-28","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"40","tokens_per_task":"51121","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.8 Extra High","source_effort":"Extra High","source_model_version":"claude-opus-4-8_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"d09e1f32a582008ac207e2313d156a4b6ecf52af0b85ded235698a5d7877031a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.4,"score_display":"59.4","source_stated_rank":35},"run_fingerprint":"dfa6ad4707943c2b0f2adefe1a513ca919af93c8d8f367eba67c242c5bd60a49","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_576644939af96eac30f5bf8d","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"b130100015ab735f81c564f5cbee47647039f724de3f5d6e813c0391d105299d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"3.15","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-05-28","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"33","tokens_per_task":"33548","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.8 High","source_effort":"High","source_model_version":"claude-opus-4-8_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"bb715649b07f58504743af7b5dbef2a89001587f935a09638f1c2702b4e200fe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.99999999999999,"score_display":"58.0","source_stated_rank":42},"run_fingerprint":"e57aa62a2ad362e92a50dc40cfa51ad98bfe376e54dbb7b9b137884ee6040d98","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"404a238be5c43a58101141e421c474229b639768f25d8778dbd171c80336bb7c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.81","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-05-28","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"32","tokens_per_task":"28384","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.8 Medium","source_effort":"Medium","source_model_version":"claude-opus-4-8_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"c00d21c714f639497fe17c1853f57925b75b29ec4f45a01587acdfc6f7bb2bac","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.10000000000001,"score_display":"56.1","source_stated_rank":46},"run_fingerprint":"8cae79437e0459ff176866e8033e15c6ed87f971e33fae0b905727ab52c613eb","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_16c9495bd6a270687b586303","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"990fb6e7edeaa870d560f62696d5258960af804fd3d38a796b2eb678bd628a6e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.02","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-05-28","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"27","tokens_per_task":"19624","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.8 Low","source_effort":"Low","source_model_version":"claude-opus-4-8_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"62e5427480ec479280ddc1a71031a3f3ec198e03dcd4b96c1a6323cea86203ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.1,"score_display":"53.1","source_stated_rank":53},"run_fingerprint":"2ccef986a1f3d75927bb28fa76eb03786e1d3db9ac0844d93ade7be7567c215e","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_889e221c2cf191b39b603fd5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":85,"at":"2026-07-12","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-12","status":"fresh"},"measurement_id":"a1ca2e0ae64710208b233966759aef3858c0bf15a056f22768a6465485309a73","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-8_unknown","source_row_date":"2026-07-12","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"d3cca460ab4bb7193a78c5fe79da02aa8e3f4b1608bc2f2d43a1e4e0f62c44dd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.06,"score_display":"47.1","source_stated_rank":1},"run_fingerprint":"52f266275837f4a034cc866670beab2aa2e92ec9894ff616be1dbcae94e1fb7f","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_362ecb5ef36351fe2feb7757","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a25e70e57d3f458b462e45462d21b4db98188151aa9fe7dd964382747c2cc7a8","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8","source_effort":"max","source_model_version":"claude-opus-4-8_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"6c43f8697f1e896c86b18155ebe54af2143529dfe05f19c2b87a6469d8be96e5","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":46.5,"score_display":"46.5","source_stated_rank":12},"run_fingerprint":"08db171bb477549abcac6083c41a4481f4f0205b8eec61ffb23e6b66050470bb","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"10ee70b20894ec021acbc93bd4916d1052f078f9c2c6083212459af35614129c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.327373,"date_is_proxy":true,"latency_seconds":159.595,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.948},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-8"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"75aa0f5d6c299be6d2438615c0141c5ab501fa210993281baa23f1255ff973c2","result":{"confidence_high":89.67708,"confidence_low":85.96092,"sample_count":null,"score":87.819,"score_display":"87.8","source_stated_rank":11},"run_fingerprint":"556bee8a50bd06b3767abd7effb811ae1ceddb878d0ce3aa1303bba1a6794017","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a238a8de3905ea236f992b571808c5af5db9e44d75c7a5693bccef204386cf55","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-opus-4-8_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"7fbb68e2e35d45d01ecc52548723a571e74566f1f545239950b3e54ac27baf7c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.4722222222222,"score_display":"53.5","source_stated_rank":62},"run_fingerprint":"855492743aaea2aad509f3d63d6560d2a76b6275461801888a12d9da137893e0","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c27ee311fd10619d9b46aa85028a44b539bc845f451d8554064bfa170a2d8ca8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.963945","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-4.8-opus-high","source_effort":null,"source_model_version":"claude-opus-4-8_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"2d92da04a443fbc506c1f2abe595602cd41f70cf835a170a1b89821baaa2b939","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1563.83,"score_display":"1563.83","source_stated_rank":16},"run_fingerprint":"8db844c147e3d6243c833c9c714f81f8003ab83a0d19fe1e47eea9d76e4c7431","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"88e9809c7b1fbe38c2d4044ae5ce351d53051de44ee5562d1c3a6f1ef46feadc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.421746","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-4.8-opus-no-thinking","source_effort":null,"source_model_version":"claude-opus-4-8_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"aaf95a646b2a84288266d6b60b2b489817564b704de0de6f79034e4f067cdf3d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1411.83,"score_display":"1411.83","source_stated_rank":21},"run_fingerprint":"ac1da7fd7d1c6ce2cff684c5dfdcd1173580332d866fc3bbd5a9c736f36929a9","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_50e3edadf9217a1e309de819","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-05-28","status":"stale"},"measurement_id":"4956757e42f6942f3996b502a84d14f281ac26feff47d08db38e8063f039ece7","metric_details":{"benchmark_page_updated":"2026-10-02","cost_per_test_usd":26.879989,"date_is_proxy":false,"latency_seconds":4549.256,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/anthropic_claude-opus-4-8","source_locator":"Results · Overall accuracy","stderr":3.081},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"0b4e8930d1d187857a6f38d43d25965bcfcfdb84c7e258c38a525acf953c59c7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-8"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"3aafabd657290682e3607ec1f742b5d5d90c27db477fe269ad048fc1e88efdc4","result":{"confidence_high":88.76375999999999,"confidence_low":76.68624,"sample_count":null,"score":82.725,"score_display":"82.7","source_stated_rank":17},"run_fingerprint":"3250ff0bee57ae38934cef61c300037658dfb1c067a58a803401fef8c9d94cfa","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"82d71180772461852a8f93d7f48853a99025d99205844786b7a3553e4bd7d752","metric_details":{"license":"Proprietary","variance":8.516614094874367},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-4-8-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b6c2dcee3694c7d9d5a5b91d30dac1934cea5157c85a9b3464ebce147d941b19","result":{"confidence_high":1561.2527342923872,"confidence_low":1549.8131150090014,"sample_count":18908,"score":1555.5329246506944,"score_display":"1556","source_stated_rank":35},"run_fingerprint":"81ba1ed27749857a6387cd6843a2da711aee30a0f5d3fa8b32cff049069bb1be","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"10c0b3cedaa8ee7e773f25784c314fff83cf3a4eb0945bb8309e77bfe758a169","metric_details":{"license":"Proprietary","variance":8.270963315069624},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-4-8","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7b0d2d0f5aa3ab9945bb0f45643cbf441f70ba9201e09717456bf3b0531ae786","result":{"confidence_high":1539.7632093607406,"confidence_low":1528.4897777656472,"sample_count":17880,"score":1534.126493563194,"score_display":"1534","source_stated_rank":43},"run_fingerprint":"0a4d6659ae0a1b2fa17745f23899890e9a84bdb6d5fc5198dc4c5abc60d7b023","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_349b6bc9c0c70c9bdf578be7","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7c11c8401169471d929e55e0574cb6e40c4dff9bbab931d8a9e9bbd7a92cf67c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-8_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"e54d5a3fb823d1f43ef214a41143b681cf7fa4f7a769f91d34e662839fef42ce","result":{"confidence_high":1565.35,"confidence_low":1552.31,"sample_count":13787,"score":1558.83,"score_display":"1558.83","source_stated_rank":29},"run_fingerprint":"45e0b7e8e6a8a53bd3acf3ab751c1e11529e5afd01fd0af37dc8520e16de1716","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"009354c482fe02f0572ad850db6a7d96fcc9a373755384f84c28726dd43dab39","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-8_none","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"1ae0b3bf96af9b2e445fdcaa161546ac500c3bb0bda8c6ac87dcd1a3ae19404d","result":{"confidence_high":1545.26,"confidence_low":1532.36,"sample_count":12654,"score":1538.81,"score_display":"1538.81","source_stated_rank":37},"run_fingerprint":"0daaf6f05f3797ab6082aedc5c3a4425d44a5f82dfbc8fb5a1927b8b3da6b17d","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_50e3edadf9217a1e309de819","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9668e12fac797078be02bea0602bed41f47a488ced57885af5bc8d987601caa2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-8_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"ec0ea67817223adbbec19d420286e42cb38bb081ea90f68a225eca204bab503f","result":{"confidence_high":1568.57,"confidence_low":1535.88,"sample_count":1578,"score":1552.22,"score_display":"1552.22","source_stated_rank":34},"run_fingerprint":"fe137edf3797451fa2f7ba5c6c3b460fb6e1c6805e88400e661019ff678313fa","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a001bc2a550e60eda54bd7b0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"671d2152da36044730cbf1ea26e6740821f60255138f73d3f1a3e7b4e5288279","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-8_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"016047521c58591a431809a45a440c6aeeadd415123b67fb28607bcac62c78a6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.89,"score_display":"82.9","source_stated_rank":16},"run_fingerprint":"c60faa22c4912c0cfc7d56774907fe0b640809694774851d503b793e4fd5b097","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_576644939af96eac30f5bf8d","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"51eb5a0c61a3c04e958e6797c78ede15c93ccf21d6ac1356fad77d0185e7cb60","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-8_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"c1a5f59bacddefa7dac56221aadafcdb45f5edfce47301d0a12811f4af8df7ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.04,"score_display":"76.0","source_stated_rank":27},"run_fingerprint":"796a8b3deea2988e0171b46278274e7bc5ef873c81784bb4855396156fff128d","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_16c9495bd6a270687b586303","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5a531c05446f69dc9316229d38daf8b2b3d2130211d732cf7d1fe565890f712f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-8_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"4e1bf1ae1c1f291de05e729ce3b0e27c5cd422aa0c911fe752d6616167862ae9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.45,"score_display":"70.5","source_stated_rank":32},"run_fingerprint":"483a3152055173d1a70e764fa847a4b1971df0e1e9a096d191e7113f81fd6f2e","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_50e3edadf9217a1e309de819","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4477d00ef21e97d9fb1589abf496b8569d0aac333d9d411c1182bf36b4a117a0","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.409772,"date_is_proxy":true,"latency_seconds":929.902,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.649},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-8"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"895fabe77050d82c8b4a048400c3dfbbcf7c2f64624df564ad22710729834f7a","result":{"confidence_high":73.18204,"confidence_low":70.63795999999999,"sample_count":null,"score":71.91,"score_display":"71.9","source_stated_rank":25},"run_fingerprint":"d7fa79d74a1c880f4a37a6a2fa52f97eafb2cf0e24748d8477256bb21fda665a","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_6d7e85c927ba2919129973e2","split_or_window":"openai-provider-comparison","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"46ed695ea7b236346079f59f893af757dbc778f52c484ce9a8a1080259aa7837","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · Terminal-Bench 2.1"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; competitor rows are secondary re-reports; exact harness and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-05-28","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"terminal agent toolset"},"run_count":null,"scaffold":"comparison harness not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"de1bf7f161a77b6573a45a5743f906805a3ebc275a61f0750f7234c0a669713f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.9,"score_display":"78.9","source_stated_rank":null},"run_fingerprint":"f0b6dde34325924a3afb627bfb2003ad73b8c0d8475b86996eede32bac77d9a8","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · Terminal-Bench 2.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_300bc637ebe4d241305bff6b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T21:39:34.661007+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T21:39:34.661007+00:00","status":"fresh"},"measurement_id":"687e9341980bd7dc7c3fb4b77dd726c93c64312a30a806f3b4271c74369b08aa","metric_details":{"accuracy_stderr":1.55,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:07:38.936797+00:00","total_cost_usd":5214.05,"total_tokens":5208776503},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"Anthropic","url":"https://www.anthropic.com"},"agent_url":"https://claude.com/claude-code","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"e728cc33-0f2b-4eda-9749-a46be31c01bb","model_release_date":"2026-05-28","model_url":"https://www.anthropic.com/news/claude-opus-4-8","n_trials":370,"source_row_date":"2026-07-12"},"run_count":5,"scaffold":"Claude Code","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"88160db45de7ebcb4aa5f678c797f77980ae783f5fe91a0902414c7a82885016","result":{"confidence_high":31.680023,"confidence_low":13.334468,"sample_count":74,"score":21.08,"score_display":"21.08","source_stated_rank":6},"run_fingerprint":"523813b6acb7d59ae02893f63a8cb297f306a5666fb62934eade13ca89100122","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":74,"at":"2026-07-23","basis":"evaluation_at","evaluated_at":"2026-07-23","first_observed_at":"2026-08-28T07:09:05Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c620f61ada1c894b37e1dcd45d66c12a3368f9df4fc41923c7888e66bd6d5044","metric_details":{"cost":null,"pass_2":31.271477663230247,"pass_3":26.288659793814436,"pass_4":22.68041237113402},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"claude-opus-4-8_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"88e9ba377425eca2d26ca6bb75fce7b15a85a4bada5fe73001024ea97041a534","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.69072164948454,"score_display":"39.7","source_stated_rank":null},"run_fingerprint":"ab8d1d73de7146767b7528771c585c08d897cc4664fbe84ace2410f84a7f4fb8","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":129,"at":"2026-05-28T17:37:01Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-28T17:37:01Z","status":"stale"},"measurement_id":"8f13ff9b4f77447323dfd3ef58f788d156101d2fc97c3de4b2c0eecccf7d79e7","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.4,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=claude-opus-4-8 (max); createdAt=2026-05-28T17:37:01Z","source_row_created_at":"2026-05-28T17:37:01Z","task_pass_coverage_threshold_percent":75},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"claude-opus-4-8 (max)","source_reasoning_label":"max","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"ae86fd7e7764158c6b38ff3f46937ba7d29350c3a615b62e8e19d82323d0e0cc","result":{"confidence_high":84.60000000000001,"confidence_low":79.8,"sample_count":1000,"score":82.2,"score_display":"82.2","source_stated_rank":2},"run_fingerprint":"4e26d4529024a02bf3a2155d08d4fc7f58f256685a38c995cdd866433b5f51d2","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=claude-opus-4-8 (max); createdAt=2026-05-28T17:37:01Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"967d0832a5742ff76f01e9f8f0cbc3a53196921113e7f79e147b0bc88db41749","metric_details":{"expected_trials_per_mode":261,"normalized_gain":15.5,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":8.4},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":62.7,"confidence_low":45.5,"sample_count":87,"score":54.1,"score_display":"54.1","source_stated_rank":7},"run_fingerprint":"b2d7a9c072e3c4e8048aa076970fc9face7a6bc18ddb0b75b2ac35eb8d095281","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_63ff087821172cc7e101cfa5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"cd5e65df3a19209f20bfa15568271212c49e6a5cc7471b96312c502bde39292e","metric_details":{"expected_trials_per_mode":261,"normalized_gain":15.5,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":8.4},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":54.900000000000006,"confidence_low":36.5,"sample_count":87,"score":45.7,"score_display":"45.7","source_stated_rank":3},"run_fingerprint":"a99a61f0b2436bef5c9563ffaab1b7e7d7b5e664e5de7494b0236a456d3fe011","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_63ff087821172cc7e101cfa5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6d08c88a1ea434973578b953bd595bf85e3e7f69c8704f89cdd594166ec9590e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.644","mean_standard_error":"4.6","model_release_date":"2026-05-28","notes":null,"standard_error_points":550.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.8","source_effort":null,"source_model_version":"claude-opus-4-8_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"7c18fd41845a1025f65b636e511f938f12826ef1e5f44e5719cd7872d11b70cb","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":48.9,"score_display":"48.9","source_stated_rank":25},"run_fingerprint":"67e2b77239e639374edfecef85c32e796e5b7ed3c6f74217eda0f2c835592850","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a85704d2ad781f5f67d2cc1be9687a8ea314c1ef25d7ea5c8a86b0c1c2aaeafd","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":41132},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Claude Opus 4.8 (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8a072ef8bf79de1e7f284c03fce576900bfe123a2637316db2ba042a2dfb3aaf","result":{"confidence_high":0.07984276590877441,"confidence_low":0.05299622583850096,"sample_count":2431185,"score":0.06641949587363769,"score_display":"0.0664","source_stated_rank":11},"run_fingerprint":"0e76f2092423c3b81f0e16614ff380c2c41908a90a62c9bcea70d3008a387d75","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"667a1feb47b32b99720ea402e48e45ff8e9781fbdf2f057c1e78190b56e606e4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 - Max","source_effort":null,"source_model_version":"claude-opus-4-8_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"c425bb83180b8a8f09c0a10f808a35acd64e6fafd4b7a0c98900a56b812fe5f8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2992.344,"score_display":"2992.34","source_stated_rank":44},"run_fingerprint":"ffbbe55cf0411a11b560e60c0d0deb228e043fea167b0d159cb7b309a3f1777e","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5b2d8e0147ca7539edd907c4b6cb4e9d710b009bdd961be66b6f001de81871dd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 - High","source_effort":null,"source_model_version":"claude-opus-4-8_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"625342c9497456e04cb3aa7c61ed76ed82c47fd7b65aaa9d5ba34f64c71dfe96","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5787.428,"score_display":"5787.43","source_stated_rank":19},"run_fingerprint":"b74ed2be1615a1d8eb4c30c1cebc40ee5c0ac102b1683cf7bfef18696e875746","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_3ff20fb03ea5572c13bd7021","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9469afc1b24f241f2a73ef3b4e6ea8f206cea019ffbe7493a22ef42b14079c7f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-05-28","reported_confidence_interval":"-21 / +21","source_locator":"Leaderboard models table · Claude Opus 4.8 (Adaptive Reasoning, Max Effort) · gdpval"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-4-8","upstream_model_label":"Claude Opus 4.8 (Adaptive Reasoning, Max Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"691084f32cc2c8e6696e192c3eff605bb65da33b6c1e833aad19859b43765bab","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1438.32,"score_display":"1438","source_stated_rank":null},"run_fingerprint":"d8cceeea18f5e096be7dcb1d6a8260ef5d2c6e9bb26a2ee65a69d45642ecfec8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 4.8 (Adaptive Reasoning, Max Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"667dfb459f3de5139314fef62fa539cb93195b83eb047b025225666993baab10","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (max)","source_effort":"max","source_model_version":"claude-opus-4-8_max","source_row_date":null,"source_scaffold":"batched tool","upstream_leaderboard_url":null},"run_count":null,"scaffold":"batched tool","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"3f78a90e46b6e5d1e3b3d6dac19d9b6b42d15334f71d453e4d086ee22bb1f582","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":20.6,"score_display":"20.6","source_stated_rank":7},"run_fingerprint":"9a0e545d2f8beae7c1797cbe4bb0960397ba0c903d496ede01ab4fee5f9ff82b","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3a0776c96ea9819d971e0f95d56ab194b511291c82865e295e2018f52e9e159b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-28","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-opus-4-8","name":"Claude Opus 4.8","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.8 (max)","source_effort":"max","source_model_version":"claude-opus-4-8_max","source_row_date":null,"source_scaffold":"standard","upstream_leaderboard_url":null},"run_count":null,"scaffold":"standard","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"c93caf81c875e3c02dcd06d63f064de33cace6fd4d031d554add20d4520e2f44","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.52,"score_display":"18.5","source_stated_rank":8},"run_fingerprint":"35192dfecb949dc080eb4245cc62973ce8e187ce7adbd2a02fe3a50ebcdf9d52","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5ce4eaa381ad965656436579","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b6419be648d0b0f6baa6b2846de71eba8dc37692a3430e1f3b3db47bb30dd2a4","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-16"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Kimi K3; model release: 2026-07-16","source_accessibility":"Open weights (non-commercial)","source_model_name":"Kimi K3","source_model_release_date":"2026-07-16","source_model_versions":["kimi-k3_low","kimi-k3_max","kimi-k3_unknown"],"source_reasoning_efforts":["low","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5e9bdf9fd84a4de28b7a78f82bce4d076d577288b2fc0cfbe962b71b5884ae29","result":{"confidence_high":160.41,"confidence_low":154.92,"sample_count":null,"score":157.61,"score_display":"157.61","source_stated_rank":13},"run_fingerprint":"08145c9693c99038587bd325bd494d93a0742588e36734e0c697c4e649155b4a","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a2da5036f8e35f4973a1f1d7","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"26870ec7ed14f3ce11b1d99cbdbb46121e82c69909716a6ac70d497b580954cf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Max) · intelligenceIndex"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3","upstream_model_label":"Kimi K3 (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"96b5d01ea4e6eab150b3c32987684c065b43a51f65db0c812653baa3beed391b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.593823,"score_display":"43.6","source_stated_rank":null},"run_fingerprint":"673a0984d37536b36721cfb8d2679da7ef3c2c53d43764f82ac305d86e49ad5a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f02056691983e811a8a47356feabad4c32ae5e36b572eb9c1bb68c502f9ed0af","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Low) · intelligenceIndex"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3-low","upstream_model_label":"Kimi K3 (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"afc313ae68ec3c9b88d5315d33615772308a3864c9431292415759343a193292","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.066699,"score_display":"30.1","source_stated_rank":null},"run_fingerprint":"b9120df136cbb8e7a34c5d1bd444bce99c12f1286a98d2b04e1ef75dacb4560d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b1175025975568042b4e3d5d28af3dfb4129d63b7eef14df45d5f3008f0883da","metric_details":{"license":"Kimi K3 license","variance":5.926917974645507},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1493.1292824239736,"confidence_low":1483.5861149554162,"sample_count":28280,"score":1488.3576986896949,"score_display":"1488","source_stated_rank":16},"run_fingerprint":"9f5f9a068b0a1b7880ac11f1c46f2ba4e8866cfbd8a00958ecf66f7f33ff0151","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"25d6c99796c42633e51e05b9d290e4434848868b23b8410f040b6a1e335dda1e","metric_details":{"category_scores":{"Agentic Coding":62.17166666666666,"Coding":81.4455,"Data Analysis":78.73366666666668,"IF":71.36275,"Language":85.528,"Mathematics":84.43675,"Reasoning":90.673}},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.19304761904762,"score_display":"79.19","source_stated_rank":15},"run_fingerprint":"5d6574e8c6a86c5e4d000504bd2559577d30e8c41c24ffc09db189e2c011dd04","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2ad36c6efaf22e10e2082fe7","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d43dc12b725969a1cb1e2a28","split_or_window":"provider-report-no-tools","unit":"percent","version":"HLE-Full (provider label; revision not restated)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":70,"at":"2026-07-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:18Z","last_confirmed_at":"2026-10-05T12:33:36Z","observed_at":"2026-10-05T12:33:36Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-27","status":"fresh"},"measurement_id":"f72fb32ca3e8153ca58cec8fc8854dca8ab7e989fc9a620b6859ff79f785c84f","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · Evaluation Results · HLE-Full first value"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Moonshot AI","evidence_verified":true,"freshness_proxy":true,"protocol":"HLE-Full provider comparison; max effort; temperature 1.0; top_p 0.95; no tools; exact dataset revision and judge not restated","reasoning_effort":"max","result_published_at":"2026-07-27","source_content_hash_method":"raw_bytes","source_content_sha256":"57de265b5842dfa465c6e73b368b0e15a89b8793b5450528dad577da202cc6fe","source_published_at":"2026-07-27","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6547241796b7257e8ab884f956a879819be7729074e34228a3dfbd69d4f79e1c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.5,"score_display":"43.5","source_stated_rank":null},"run_fingerprint":"e35a8b6faf99dbf6a2688cb01c029a24c7760cbb408df0c9348eeef2bd86d431","source":{"content_hash":"a4cfbbf5b6a39cd0579b65c1b7fc3bfa4f21bd4f4183b560ba850a8a6ccdfca7","fetched_at":"2026-10-05T12:33:36Z","first_fetched_at":"2026-09-03T09:21:21Z","homepage_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md","id":"moonshot-kimi-k3-card","last_fetched_at":"2026-10-05T12:33:36Z","license":"Kimi K3 License; minimal factual score fields with attribution","locator":"Immutable README · Evaluation Results · HLE-Full first value","name":"Moonshot AI · Kimi K3 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9c68d2033132902c39172f233ea105438fa884a967867f346c24c5fc59777d4e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":null,"num_parameters":2779931837184,"pull_request":null,"result_file_url":"https://huggingface.co/moonshotai/Kimi-K3/resolve/main/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/moonshotai/Kimi-K3"}},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.0,"score_display":"56","source_stated_rank":4},"run_fingerprint":"fa2bed851d541dff4247f76484687b386c711a48a0c7c5140ee0dcb5eaeae8e4","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e57858e1524329598a68c6b1","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_89ec793730446cc789bf9c2b","split_or_window":"provider-report-with-tools","unit":"percent","version":"HLE-Full (provider label; revision not restated)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":70,"at":"2026-07-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:18Z","last_confirmed_at":"2026-10-05T12:33:36Z","observed_at":"2026-10-05T12:33:36Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-27","status":"fresh"},"measurement_id":"a789b5004cdd27b7fab16d03ffd5345a8692dae577ab5b9f009f4ed64ac20d77","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · Evaluation Results · HLE-Full second value"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Moonshot AI","evidence_verified":true,"freshness_proxy":true,"protocol":"HLE-Full provider comparison; max effort; tools enabled but exact tools, dataset revision and judge not restated","reasoning_effort":"max","result_published_at":"2026-07-27","source_content_hash_method":"raw_bytes","source_content_sha256":"57de265b5842dfa465c6e73b368b0e15a89b8793b5450528dad577da202cc6fe","source_published_at":"2026-07-27","tools":"tools enabled (unspecified)"},"run_count":null,"scaffold":null,"tools_allowed":"tools enabled (unspecified)"},"protocol_fingerprint":"12d27a241fdafc90677ed16993b87493147ecceb7cbf46f0d49349083d269360","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.0,"score_display":"56.0","source_stated_rank":null},"run_fingerprint":"32d552df71885ed97966bd336649b36501e813774d33956597499f998ff58aea","source":{"content_hash":"a4cfbbf5b6a39cd0579b65c1b7fc3bfa4f21bd4f4183b560ba850a8a6ccdfca7","fetched_at":"2026-10-05T12:33:36Z","first_fetched_at":"2026-09-03T09:21:21Z","homepage_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md","id":"moonshot-kimi-k3-card","last_fetched_at":"2026-10-05T12:33:36Z","license":"Kimi K3 License; minimal factual score fields with attribution","locator":"Immutable README · Evaluation Results · HLE-Full second value","name":"Moonshot AI · Kimi K3 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bf5b4f9b3ac5a5b2bc629563cbf514d2d445fd3188ee9bf22f70c8a917581d2b","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Max) · hle"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3","upstream_model_label":"Kimi K3 (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5f186d3b086a86c110db812809bdc24ac09d3c08cf0889f22d5c3cdc81a3ef24","result":{"confidence_high":49.004456,"confidence_low":44.797124,"sample_count":2158,"score":46.895273,"score_display":"46.9","source_stated_rank":null},"run_fingerprint":"75ab1935ae582569ec6d533d127ab15066179c87b0c96000f719fcfdc49ed839","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9fe346bc9295f6eb614da40fcae538fb7c67e22267a7ea867d86755c245ffb6a","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Low) · hle"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3-low","upstream_model_label":"Kimi K3 (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f7d39c2b97ec8f64b70923d68241346fcbd71ea1002d935e37294e30e206f9af","result":{"confidence_high":26.846618,"confidence_low":23.195975,"sample_count":2158,"score":24.97683,"score_display":"25.0","source_stated_rank":null},"run_fingerprint":"1332b034d18c608345533396fec9c7e60828564055a191bc65632cfd63d6e007","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:36:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:36:12.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0b67789d3cf0cc2cfabed4d1d8a1b429c057ad7ac32801c30ee5c2701324dc08","metric_details":{"best_score_across_scorers":"0.506","model_release_date":"2026-07-16","stderr":1.5818160898606837},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mZLWt2JSYvJyGhMvji7CT8","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmZLWt2JSYvJyGhMvji7CT8.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mZLWt2JSYvJyGhMvji7CT8.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"8a3839bf844fafb00603a5223443e491cfff9eb6b96d800ff480fcd85634633a","result":{"confidence_high":53.70035953612694,"confidence_low":47.499640463873064,"sample_count":1000,"score":50.6,"score_display":"50.6","source_stated_rank":24},"run_fingerprint":"974a1f51eb5c225e7487e2d30f62f1c49253058dbc7e3fc0865b6ad0d40817dc","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1ff642173452f42a62638f557d9ea1b6afb1390e2189c1176b1cc519de9774bb","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.088491,"date_is_proxy":true,"latency_seconds":122.425,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.309},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":93.434,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":262144,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"kimi/kimi-k3","zero_shot_accuracy":92.424},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"b9e379937371c4fd681ca77eece8201794aaf66c2d6c8a62b09568ac52caec3b","result":{"confidence_high":95.49464,"confidence_low":90.36336,"sample_count":396,"score":92.929,"score_display":"92.93","source_stated_rank":12},"run_fingerprint":"05e1e4c9c95f4201f7441f8af586f2277c92d13eb240761b5092b1f75287b790","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1e5c3455b6099d35084a3c99","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7921f01abebb49cc2fc9430988f71c0cfefe3c0596dc5e6bc41c708d136204ab","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Max) · gpqa"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3","upstream_model_label":"Kimi K3 (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9f787528aa1e9f5c2125b575ef5527a33c16075bdb9bb28a0d1909c1d9b45584","result":{"confidence_high":96.198914,"confidence_low":89.2146,"sample_count":198,"score":93.535354,"score_display":"93.5","source_stated_rank":null},"run_fingerprint":"e087e7c2075a27fd350e11a3c997226ad7e35e9c118b4a59590d51fe678357a7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Max) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dfd9ad2a17ad7375132860e999400e15aca900e06dafcd5b34bd14fc7e6719cf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Low) · gpqa"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3-low","upstream_model_label":"Kimi K3 (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b9a0b5e47c7b6e753b57eff9b584cb92ab60ab7117d18a4e9aa3e87a535512f4","result":{"confidence_high":88.659216,"confidence_low":78.522177,"sample_count":198,"score":84.242424,"score_display":"84.2","source_stated_rank":null},"run_fingerprint":"4a5c9380267f2f25b24e994a738011660d9f53c0b4dacd273be4436fc700cd18","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":80,"at":"2026-07-16T21:59:40.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-16T21:59:40.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"64aa52cbcb9b8ec4b230ebce63f0c7424a20631b5045a44397fb404340f13be5","metric_details":{"best_score_across_scorers":"0.9311868686868687","model_release_date":"2026-07-16","stderr":1.493722536419486},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"B8MaPUe3EYwUZASsNqE7C9","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6a25fe9e9581e0bb6881a627e6125431184bd739da7432976766a953389748d5","result":{"confidence_high":96.04638304006907,"confidence_low":90.19099069730468,"sample_count":null,"score":93.11868686868688,"score_display":"93.1","source_stated_rank":20},"run_fingerprint":"3526332aea532344a67753125d772c85bc58b4ae693acb20930d16fbee10a991","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:53:51.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:53:51.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e9c4902685d0bb71251d1c779bc2236f3dd6af8fca901c911ade9d742df57d90","metric_details":{"best_score_across_scorers":"0.9191919191919192","model_release_date":"2026-07-16","stderr":1.9417681889724516},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"AoAD5uVtgVDnXmwA7STuno","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAoAD5uVtgVDnXmwA7STuno.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/AoAD5uVtgVDnXmwA7STuno.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d460a27bc4fcb0a3509a88e39d2a0fe83dc871309dc31ee46558636753e44f50","result":{"confidence_high":95.72505756957793,"confidence_low":88.1133262688059,"sample_count":null,"score":91.91919191919192,"score_display":"91.9","source_stated_rank":27},"run_fingerprint":"7fca87a9c817272f0ba132ac8a098a353fa2e357b17bda739727b334f185dde1","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1940792474091774d582a01e","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:54:16.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:54:16.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e9b2432d4adabdc5ea55a4531d98d4fdd0c02d17f9772fc9670e4ad91c3dec6e","metric_details":{"best_score_across_scorers":"0.8484848484848485","model_release_date":"2026-07-16","stderr":2.554565042660364},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mvNb3hNgSZ2gS6jHMtvCdk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmvNb3hNgSZ2gS6jHMtvCdk.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mvNb3hNgSZ2gS6jHMtvCdk.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"492ed240323cd40165cdca3c5fad9f3fee10d3a179050f11d18587d243fa0260","result":{"confidence_high":89.85543233209916,"confidence_low":79.84153736487053,"sample_count":null,"score":84.84848484848484,"score_display":"84.8","source_stated_rank":92},"run_fingerprint":"47adf27dad4f05cbc66ba1048d9dda0c94efed5b8c43a932d737f19dae851b86","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_6582102e5715be19a872de80","split_or_window":"provider-report-no-tools","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":70,"at":"2026-07-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:18Z","last_confirmed_at":"2026-10-05T12:33:36Z","observed_at":"2026-10-05T12:33:36Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-27","status":"fresh"},"measurement_id":"b49230591076875d979685640942e6c655122d04dc2fceb5b2009c4e75ca7ec5","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · Evaluation Results · Reasoning & Knowledge"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Moonshot AI","evidence_verified":true,"freshness_proxy":true,"protocol":"max effort; temperature 1.0; top_p 0.95","reasoning_effort":"max","result_published_at":"2026-07-27","source_content_hash_method":"raw_bytes","source_content_sha256":"57de265b5842dfa465c6e73b368b0e15a89b8793b5450528dad577da202cc6fe","source_published_at":"2026-07-27","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8f7f5233969bb1163c4f1c56185232f1ba3fea8929ab8b5ba57de5dd61572fc5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.5,"score_display":"93.5","source_stated_rank":null},"run_fingerprint":"5154cee92f75a261d94407b560fb17d35226231c67362ce0fbfe3f2c9012b6fd","source":{"content_hash":"a4cfbbf5b6a39cd0579b65c1b7fc3bfa4f21bd4f4183b560ba850a8a6ccdfca7","fetched_at":"2026-10-05T12:33:36Z","first_fetched_at":"2026-09-03T09:21:21Z","homepage_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md","id":"moonshot-kimi-k3-card","last_fetched_at":"2026-10-05T12:33:36Z","license":"Kimi K3 License; minimal factual score fields with attribution","locator":"Immutable README · Evaluation Results · Reasoning & Knowledge","name":"Moonshot AI · Kimi K3 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c45f88f993e60eed791a3ebb8dd485f4d5ca189fbd405e238634dd3f15435d93","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.021327,"date_is_proxy":true,"latency_seconds":38.718,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.321},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":262144,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"kimi/kimi-k3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"bfb435f6d5f4e52f842b42355538a2e623e9ba99464174516e6592cebb3d470e","result":{"confidence_high":88.60216,"confidence_low":87.34384,"sample_count":null,"score":87.973,"score_display":"88.0","source_stated_rank":22},"run_fingerprint":"a01bb582f83d7b17d3755166fc7a14390199a0ad4762ab27dc1d5d21ddb25378","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":80,"at":"2026-07-16T21:59:40.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-16T21:59:40.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7a6e86b6e615837f44f0a7a1cf839ce2bb0373966370f4d1b524798766c22b00","metric_details":{"best_score_across_scorers":"0.9722222222222222","model_release_date":"2026-07-16","stderr":1.1158359139033183},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"A33om9MTYMC7xYeLv9X7Ha","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5083290679b6574e185789e24ba694af1cd7e60a77fa6d03706ab192a8c43cb9","result":{"confidence_high":99.40926061347272,"confidence_low":95.0351838309717,"sample_count":45,"score":97.22222222222221,"score_display":"97.2","source_stated_rank":31},"run_fingerprint":"2ff2230b716ef41029ad52eb3739e02240e495ceb575b4cb8a2b8be3621faa54","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:46:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:46:05.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"39dde82d3fa9fd4a0bb90e389b209a4fed540420fcf761a720ed29f51ad99bf0","metric_details":{"best_score_across_scorers":"0.9333333333333333","model_release_date":"2026-07-16","stderr":3.760507165451775},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4E5xGsuRMwMBhAfC4F2G4N","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4E5xGsuRMwMBhAfC4F2G4N.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4E5xGsuRMwMBhAfC4F2G4N.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9ab44996256320db85ab939070f49d4f099e67099ff365cc9bce550502324a6e","result":{"confidence_high":100.0,"confidence_low":85.96273928904785,"sample_count":45,"score":93.33333333333333,"score_display":"93.3","source_stated_rank":53},"run_fingerprint":"69d6f444ed1fc00838f22892bed514990d799ee93496e58b7adcf5420dcc7240","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1940792474091774d582a01e","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:46:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:46:06.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6d0aca55bdc40d5cc94108a920d85c6741bea199a6f94e7fc6b54fc9b9cfaeab","metric_details":{"best_score_across_scorers":"0.6888888888888889","model_release_date":"2026-07-16","stderr":6.979205927323111},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"frszScNxpHXXjSHawCt9Zb","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfrszScNxpHXXjSHawCt9Zb.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/frszScNxpHXXjSHawCt9Zb.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"346d5da56075a3d7487d341a72c6ac46d3734240fee4ded4a9e6049ba6c535ad","result":{"confidence_high":82.56813250644218,"confidence_low":55.20964527133559,"sample_count":45,"score":68.88888888888889,"score_display":"68.9","source_stated_rank":140},"run_fingerprint":"1654eaa56885f95f6a558b70d9465e612db6d66d4b02b23123e6ac1d6bb1c4c7","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":80,"at":"2026-07-17T01:48:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-17T01:48:42.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6704cfa59f4bafc63e0352a918f16e2984877091db15791722054de9837bd704","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.721830985915493","leaderboard_private_problem_count":285,"model_release_date":"2026-07-16","public_example_count":10,"stderr":2.6636607935112644},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"bcntamvGsSH6QgETdoaSph","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Kimi K3 (max)","thinking":null,"upstream_model_id":"kimi-k3_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"8cd6dd3d3cb2d972401111c3a3dbca3df50385d3b1f06d522a56977e1ea3578c","result":{"confidence_high":77.40387374683138,"confidence_low":66.96232343626721,"sample_count":285,"score":72.1830985915493,"score_display":"72.2","source_stated_rank":22},"run_fingerprint":"391195bf71d496c6d483f8e21f03b9f9c4db549d52acd9c84acb26398ccae537","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"90dcef2fa7cfd577245c8fc4e543a26e1ca95f227308b877d34c7ba3b0c27c68","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.5926940250000003,"model_release_date":"2026-07-16T00:00:00.000Z","results_url":"https://arcprize.org/results/moonshot-kimi-k3"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"moonshot-kimi-k3","model_type":"CoT","upstream_model_id":"moonshot-kimi-k3-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"42b22b8742fff641dc193a46a202bc267ec6e13ac30cbe53bf8a3e841fe66b6d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.416666666666664,"score_display":"60","source_stated_rank":76},"run_fingerprint":"e193c56d3c1c2306948b622f5ab7cd31152004e39b2593d6d4b92aa237ed8b24","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"98870570fc16a8d12d0ebc2cc296ea50cb137258c3f27de9b57f069b1b8016fa","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.9472229249999998,"model_release_date":"2026-07-16T00:00:00.000Z","results_url":"https://arcprize.org/results/moonshot-kimi-k3"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"moonshot-kimi-k3","model_type":"CoT","upstream_model_id":"moonshot-kimi-k3-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4ea65837445d43bc992b47690c520dad9fa25b7c61b1dab30c02aff5c1d39446","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.00000000000001,"score_display":"55","source_stated_rank":88},"run_fingerprint":"c602aff66eacf73c93177731ec33a82684322f37e7febee33205745448e72311","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1940792474091774d582a01e","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7acba77d01f1487e9315780fc74ea3bd01f9136510f3afc6ef75c341992b35a8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.2541386750000001,"model_release_date":"2026-07-16T00:00:00.000Z","results_url":"https://arcprize.org/results/moonshot-kimi-k3"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"moonshot-kimi-k3","model_type":"CoT","upstream_model_id":"moonshot-kimi-k3-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b866d9c92d6883d4f2749303a3f18a1c7babdf3cd4348dd238ed5bafe27ebdaf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":12.36111111111111,"score_display":"12","source_stated_rank":141},"run_fingerprint":"d4f1a59f0c784587441bc4e0ae96c4fcec3ddabee9cf3a1f98cd2825f7456966","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"21e8ae91ffe29269aa8a49d7d807d1133d3c4ca60ea5367b5b28c9b9a55845f7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.5926940250000003","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (Max)","source_effort":null,"source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0b71e6dc33160b4e38344b3961897ea6e36967d8f04aaa93464d165e2ae6ee1f","result":{"confidence_high":65.332868,"confidence_low":55.280498,"sample_count":360,"score":60.416666666666664,"score_display":"60.4","source_stated_rank":79},"run_fingerprint":"b6c71173ac7ceabcf411db37b5064a442f07a80552e0087778044792dd4f751d","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d244495c8b1633fdf110eb893ebd9574ec2722dc924e564ffbf90c90fdd086f5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.9472229249999998","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (High)","source_effort":null,"source_model_version":"kimi-k3_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a75f264e0c4bb612c74577f3acee687a942ccfdd3192080993b6cba44095e30b","result":{"confidence_high":60.059441,"confidence_low":49.834975,"sample_count":360,"score":55.00000000000001,"score_display":"55.0","source_stated_rank":90},"run_fingerprint":"ccfe53e8c7cd2ef38f0e2c88dc3e711e0cbcb57843bc8c9a71938648738db842","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1940792474091774d582a01e","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2bf5456c3b6638ead98a0e99cf9a93d2ef06cf5585ca19f749c6dfc096f42ad8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.2541386750000001","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (Low)","source_effort":null,"source_model_version":"kimi-k3_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"69950e8b20d4fcab1a31d0a0a72c2cd8bfb7d3d4a642058aec3dd2e260d8ed77","result":{"confidence_high":16.163815,"confidence_low":9.353223,"sample_count":360,"score":12.36111111111111,"score_display":"12.4","source_stated_rank":137},"run_fingerprint":"36229803b17191fdf6167561ce71cff402ba7aca8e266cf890741daa9adb8e22","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":80,"at":"2026-07-17T01:48:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-17T01:48:42.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"38dbdaebcc30d4f99a2886cfc973cc37810c123da86c36e8fda84db6995e306a","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.3902439024390244","leaderboard_private_problem_count":41,"model_release_date":"2026-07-16","public_example_count":2,"stderr":7.712872341874095},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"gZERSAQwxBAevQzBsWfUie","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgZERSAQwxBAevQzBsWfUie.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/gZERSAQwxBAevQzBsWfUie.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Kimi K3 (max)","thinking":null,"upstream_model_id":"kimi-k3_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"6a845d022df3285937b93ad8eb42b1c9fe0becf0699e3a92964e5c4b4d9c122d","result":{"confidence_high":54.14162003397566,"confidence_low":23.907160453829214,"sample_count":41,"score":39.02439024390244,"score_display":"39.0","source_stated_rank":27},"run_fingerprint":"bd088f532b33e9195bc04a4b98683f21bb50305aa2b9df690e8de432207325cd","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"989620e40fb6825c3cce37e44d9ac7deb74ff844d888a89d3b0f6b604950e6c6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.059115,"date_is_proxy":true,"latency_seconds":106.516,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.777},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":262144,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"kimi/kimi-k3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5b763d2e58a4d6d2078cd6fe2218ba4af31d0cad13f1919e60d5c25acc8a5b1f","result":{"confidence_high":89.67292,"confidence_low":86.62708,"sample_count":null,"score":88.15,"score_display":"88.2","source_stated_rank":11},"run_fingerprint":"4921b6bfb799169facf94cfcc80686fde1aee8f4e7b0c7874d0e5bf764d5eb92","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f82da36312f78d41c5384ff8","provider_config":{"source_id":"vals-mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b2bbf1b846083c47abff11dddb0a88a121e0c836feb30f6ca7d3df35aab244bd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Max) · mmmuPro"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3","upstream_model_label":"Kimi K3 (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"039b7ce439c0d984d049a8f5ec3c90f4a45913a980377e833fcedea814521b6e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.520231,"score_display":"80.5","source_stated_rank":null},"run_fingerprint":"0b90ad6171ccfabe4877fdf0a2b948b7c5facf59c248abf5a159dc22864ea1ef","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Max) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"10cd325fbf881d9f5808bb8efbfb548c524a63caf4bbfd8a5cb9a383732f7def","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Low) · mmmuPro"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3-low","upstream_model_label":"Kimi K3 (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a48516f8e2fb940fff925845472872139425be27a00dc9b1100f237017cfed9a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.49711,"score_display":"78.5","source_stated_rank":null},"run_fingerprint":"d268b8d410e51fe71ca748f1b950888d980360c0327b6a8e4d1f9461c421333d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_39a5bff9a48a29bea5ce760d","split_or_window":"standard-no-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":70,"at":"2026-07-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:18Z","last_confirmed_at":"2026-10-05T12:33:36Z","observed_at":"2026-10-05T12:33:36Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-27","status":"fresh"},"measurement_id":"8b44bac028804bb198173ef671ffe83abd387b73f048bb426029d38bb7ebb63e","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · Evaluation Results · MMMU-Pro first value and footnotes"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Moonshot AI","evidence_verified":true,"freshness_proxy":true,"protocol":"official protocol; original input order; images prepended; no tools; three-run average","reasoning_effort":"max","result_published_at":"2026-07-27","source_content_hash_method":"raw_bytes","source_content_sha256":"57de265b5842dfa465c6e73b368b0e15a89b8793b5450528dad577da202cc6fe","source_published_at":"2026-07-27","tools":"none"},"run_count":3,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c27fce5cf9f4f5e1bb5bc1d6b4a77ee4760bbe6aa3fbdecf70acb00bedfefce9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.6,"score_display":"81.6","source_stated_rank":null},"run_fingerprint":"9c546885ba0282d067fb00b6f640aca2e550f48f7b0b3bd28db803ba7be07c9e","source":{"content_hash":"a4cfbbf5b6a39cd0579b65c1b7fc3bfa4f21bd4f4183b560ba850a8a6ccdfca7","fetched_at":"2026-10-05T12:33:36Z","first_fetched_at":"2026-09-03T09:21:21Z","homepage_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md","id":"moonshot-kimi-k3-card","last_fetched_at":"2026-10-05T12:33:36Z","license":"Kimi K3 License; minimal factual score fields with attribution","locator":"Immutable README · Evaluation Results · MMMU-Pro first value and footnotes","name":"Moonshot AI · Kimi K3 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_7cc9414e9c26ab3f27633878","split_or_window":"standard-with-python","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":70,"at":"2026-07-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:18Z","last_confirmed_at":"2026-10-05T12:33:36Z","observed_at":"2026-10-05T12:33:36Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-27","status":"fresh"},"measurement_id":"ccaaf9a4c59a8562ec83253151ae7e49f6bef41e665e966f8e5ea84f34c5a89b","metric_details":{"comparison_warning":"MMMU-Pro with Python support; do not compare directly with no-tools runs. The report date is used as the freshness proxy.","comparison_warning_code":"mmmu-python-report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · Evaluation Results · MMMU-Pro second value and footnotes"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"mmmu-python-report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Moonshot AI","evidence_verified":true,"freshness_proxy":true,"protocol":"official protocol; original input order; images prepended; Python augmentation; three-run average","reasoning_effort":"max","result_published_at":"2026-07-27","source_content_hash_method":"raw_bytes","source_content_sha256":"57de265b5842dfa465c6e73b368b0e15a89b8793b5450528dad577da202cc6fe","source_published_at":"2026-07-27","tools":"Python"},"run_count":3,"scaffold":null,"tools_allowed":"Python"},"protocol_fingerprint":"a3aadc6938189c1a50b680bca6624b173bab57a11ca94550a2e6c89745ad3418","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.4,"score_display":"83.4","source_stated_rank":null},"run_fingerprint":"d2b6716c4c2b95aa9093756b3c446d56955830f3575ffb3b963bf42ba9478b14","source":{"content_hash":"a4cfbbf5b6a39cd0579b65c1b7fc3bfa4f21bd4f4183b560ba850a8a6ccdfca7","fetched_at":"2026-10-05T12:33:36Z","first_fetched_at":"2026-09-03T09:21:21Z","homepage_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md","id":"moonshot-kimi-k3-card","last_fetched_at":"2026-10-05T12:33:36Z","license":"Kimi K3 License; minimal factual score fields with attribution","locator":"Immutable README · Evaluation Results · MMMU-Pro second value and footnotes","name":"Moonshot AI · Kimi K3 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4114f605d96fe8a88fb1bfd63c5c6febb57fca128ae67828bb7dc4e804958c96","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3","source_effort":null,"source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"59cca39d88f18559fc0d18a44f486256be02cce06212032473cfc39f9b4d867f","result":{"confidence_high":34.45401,"confidence_low":15.076732,"sample_count":71,"score":23.400000000000002,"score_display":"23.4","source_stated_rank":44},"run_fingerprint":"a488ee6d7c20f86ead81ac5ecc026a5fdf19c37b59cdf8a2c45f586489bfa700","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6cb6b65e01204b75c888000c0f61f34807f4b286692641c680671a61771ee0f9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (low)","source_effort":null,"source_model_version":"kimi-k3_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"80df4d6bdb657636e3f2e214c68e1428cfcf340539dd1e4134bc16535ff055fe","result":{"confidence_high":10.175118,"confidence_low":0.920927,"sample_count":71,"score":3.14285714285714,"score_display":"3.1","source_stated_rank":121},"run_fingerprint":"c85666de87e3268f24a68314d6ca011c9db44053b35e2023fc1d5e4e38320124","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f5738ea2d568c6e628e874383b1f564921a084caf156ff9b01583b9ddedd63a2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Max) · critpt"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3","upstream_model_label":"Kimi K3 (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"baa49dece0f31935c2cbe51bf4ba4d6a03eedfa8881cd8870879faacdd6c1bfe","result":{"confidence_high":34.57012,"confidence_low":15.051773,"sample_count":70,"score":23.428571,"score_display":"23.4","source_stated_rank":null},"run_fingerprint":"2cc3fc7b872567d0c7ded3aa896ee211653f9b17e5ac6d56cfa9babae9535957","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"950f5887696630f2848467c7b893c55f26a7904c0a2eb91ba811f52f39f73ae7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Low) · critpt"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3-low","upstream_model_label":"Kimi K3 (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"4eb96ac688ae768ae9855dfd08b25d7992ae164aa12c01c6d84ef049b16d489a","result":{"confidence_high":10.24743,"confidence_low":0.913758,"sample_count":70,"score":3.142857,"score_display":"3.1","source_stated_rank":null},"run_fingerprint":"66265f25358a8299de86ba7aea330d4dec1aa67c6edf3b59af2a4300600f052f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a91c94b48b5a1adf6dbb39e2e39708521857fb793e05a6ce41fd164301a8719b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"34d15d8fb21b57acc8ecf80c7513e58a21cd3eea609126bf88cf0714279cba0e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.70000000000001,"score_display":"60.7","source_stated_rank":29},"run_fingerprint":"adf531b2b018a58e349453baffea478cacbc1de8cd325355e9289ac0bb85f42e","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":80,"at":"2026-07-16T21:59:40.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-16T21:59:40.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"541089e63e2f8daa5229308162aece77724d6556018540978e60ceb891918e1e","metric_details":{"best_score_across_scorers":"0.39","model_release_date":"2026-07-16","stderr":4.902071300001973},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"35hRAbkceJM3Dpb2pWXnx9","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8f3612ae138f799fe58c1808a84c4fb88a66f92269dd423d3d004a5fb93834cb","result":{"confidence_high":48.60805974800387,"confidence_low":29.391940251996132,"sample_count":100,"score":39.0,"score_display":"39.0","source_stated_rank":31},"run_fingerprint":"4f90c3fbedf5f5fbfb63f75c85d9c0b50debf71837227aec8fb4204f20cf8c6d","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:57:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:57:00.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"216e4641589f96b1ba7eedd9ca630b56dcf3d084a25149db72c14dc5637f732d","metric_details":{"best_score_across_scorers":"0.25","model_release_date":"2026-07-16","stderr":4.351941398892446},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"d8egYFK9XNSEHWvv4JFxWx","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fd8egYFK9XNSEHWvv4JFxWx.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/d8egYFK9XNSEHWvv4JFxWx.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4030adf28ef3253f042402d83923d6d5ef2e64d2c6b5d3bbf1e88013f5c19a04","result":{"confidence_high":33.52980514182919,"confidence_low":16.470194858170807,"sample_count":100,"score":25.0,"score_display":"25.0","source_stated_rank":66},"run_fingerprint":"2e6d68a5335ffb114df1f1db7208087305ce6e06865630f0ecd171b2d45e3084","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1940792474091774d582a01e","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:57:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:57:00.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"84775483c00f86bcf7dbc409abb3717bb0caa5156500820c39f7b1c0ae6e245f","metric_details":{"best_score_across_scorers":"0.2","model_release_date":"2026-07-16","stderr":4.020151261036848},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BUBiYK8WB8XDb9u7s9kTDf","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBUBiYK8WB8XDb9u7s9kTDf.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/BUBiYK8WB8XDb9u7s9kTDf.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"22557b5d425c447de0ed32648ad48f2e553f5fe40e81e9fd9f21dae0c1418e62","result":{"confidence_high":27.879496471632223,"confidence_low":12.120503528367777,"sample_count":100,"score":20.0,"score_display":"20.0","source_stated_rank":87},"run_fingerprint":"0f9312a8ed866e7178651814cae39c0283c9811561d8e9f4aa92f49c60bda35b","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ff6969933d486aea7982d06cb3630189e0efe5fc74e577e45fb2bc952d407ced","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.7704815999999995","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (Max)","source_effort":null,"source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c0f8b1c6d5c3d190821c86efc20cca9fcd013ca7bf67de5beaa04c177ae4459a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":94.5,"score_display":"94.5","source_stated_rank":37},"run_fingerprint":"57d75899ed2ef367e187a9c4573189947d1dcf2366cd224e9da01e7de6d73a26","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"78b4436c99db04acf5e2838ae1289f28311e42d80d50587bc677f22cb8fcac1f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.47977101000000005","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (High)","source_effort":null,"source_model_version":"kimi-k3_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"38e5f216d656998668e4946b01b4959b002b9a3427f1922adf3966ecd4a03d61","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.66666666666666,"score_display":"86.7","source_stated_rank":93},"run_fingerprint":"25375fbbe0d3d917333111f3fdbab8b5926206b9de08df9d7ed4269b05339a77","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1940792474091774d582a01e","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ff2329e297f58da511d3d4e139133898a4d5d24e19f21b6df01d932626d04868","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.18061866","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (Low)","source_effort":null,"source_model_version":"kimi-k3_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f3fb1529595b85f940565048c1022803991f5503607668d3c77672e4afc6b703","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.66666666666666,"score_display":"65.7","source_stated_rank":141},"run_fingerprint":"6999841fd90218991333f7549015e658ce9ad93bcf8e03ba2cdf6f296bb3c05d","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a304f230c48bcef5292f4072fb665841c4397a494c53cb9a080c66f0d6697a81","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.759959,"date_is_proxy":true,"latency_seconds":619.492,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.111},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":262144,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"kimi/kimi-k3"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"d4062e614674f5d8969913dcd09f5626d17c2a2a2faffa300cc3dfe1e4024567","result":{"confidence_high":95.57756,"confidence_low":91.22244,"sample_count":500,"score":93.4,"score_display":"93.4","source_stated_rank":8},"run_fingerprint":"8e934924b272ab3ec9253e2ac858dda8b0cb592362dadbfcdc9049fa4b48b509","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cb71961e616ff6ab8428806f","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"c25ce5401f264cfdc37f52a96502befc2ac014b482503be496e2a1d189bed513","metric_details":{"ci_attempted":451,"ci_half_points":4.53701407928232,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":309,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":97.58758314855876,"mean_cache_tokens":9501527.498891352,"mean_cost_usd":4.654682129933482,"mean_duration_seconds":4541.447376412417,"mean_input_tokens":9695436.246119734,"mean_output_tokens":81499.84257206209,"median_agent_steps":88.0,"median_cost_usd":3.6024987,"median_duration_seconds":3972.857633,"median_input_tokens":6930356.0,"median_output_tokens":75383.0,"median_output_tokens_to_pass":74586.0,"median_peak_context_tokens":130945.0,"n_attempted":451,"n_passed":309,"n_tasks_attempted":113,"n_tasks_passed_any":101,"pass_at_4_points":89.38053097345133,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_kimi_k3_max","source_model_version":"kimi-k3","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"306487a785d65dc89a1708ac4b32950b6ec9585e38149e470860ea5d905e336b","result":{"confidence_high":73.05142649613376,"confidence_low":63.97739833756913,"sample_count":451,"score":68.51441241685144,"score_display":"68.5","source_stated_rank":19},"run_fingerprint":"7e47747a62e403f96ee540c24b9dcb6238b490abd78f21d7a73484b9d6d730f9","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c972b944248b1b7c167ccb1294085abf499b197c2eea2b34c3fa1d4252a9de9c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"97.58758314855876","mean_cost_usd":"4.654682129933482","mean_output_tokens":"81499.84257206209","model_release_date":"2026-07-16","notes":null,"pass_4":"0.8938053097345133","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"kimi-k3 (max)","source_effort":"max","source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"4ec8382404184b1022d34bb09013815e3316a44848b626d75f43b4c5f4e9fca3","result":{"confidence_high":73.05142649613376,"confidence_low":63.97739833756912,"sample_count":113,"score":68.51441241685144,"score_display":"68.5","source_stated_rank":19},"run_fingerprint":"a4b09f3f61930aa3d047a067713a7f0e1762e3c601929b07f4bbaacf97ea20e5","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"92a1b3b8cbf391692cccfdc071dbdd283b694ba0c7e752f6ab5b6f43307088ba","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.7","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-07-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"57","tokens_per_task":"38428","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 Max","source_effort":"Max","source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"b3d7762433fed795dbbf314286ec5d55ffc0d06ec495b0f4f4778b427edad3e4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.8,"score_display":"60.8","source_stated_rank":32},"run_fingerprint":"ed481ade241d3bf1d5f8dbff8ecbd4d26157b5e0090decb1262d6a35a4021d1d","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"fcd75d655dfb7a22ca9040349f4dbd5ac873df9a58ca5c950c6e18828b2fdb6f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.89","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-07-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"47","tokens_per_task":"26846","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 High","source_effort":"High","source_model_version":"kimi-k3_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"fec82ca4629630e3c95fcdd99061d6f0d6e086cbfcd5ce9e5b0016c8093effc4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.70000000000001,"score_display":"59.7","source_stated_rank":34},"run_fingerprint":"c421f60b3bfdb733695ecf5e1a44f15c8642d20224fb44702b656dd48b681ec5","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1940792474091774d582a01e","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"f22076ff08710586462bb6121cc0874d7c9897999164e924767ed84389c48451","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.99","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-07-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"33","tokens_per_task":"13007","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 Low","source_effort":"Low","source_model_version":"kimi-k3_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"48c005789389da4c9186eaf68d3670939a95c5799096c5a85b2abbbb7a05c019","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.5,"score_display":"50.5","source_stated_rank":59},"run_fingerprint":"152b5c2c9c79a7b511cc8aadc17bb561c7fc309def91cc2c5ddccfc519a56f31","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4c30895858e3ce8a74276e9efb6e6c9b68578aa3263398db469559355400a24f","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3","source_effort":"none","source_model_version":"kimi-k3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"c7e0ed9a1238dc923792161e4c4e2f6758900ca23d2261dfff6b838e57b05782","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":44.17,"score_display":"44.2","source_stated_rank":13},"run_fingerprint":"23ea9ed3a0c23a506599ba1d78fc69f35d1500dc44622d810776f66c24cd4a53","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_1ae7fb380652700fc7c15f73","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"361204566d4a77f5ec4e1048c2d377936da463ed6b1df4204cb3be0984876919","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":14.165034,"date_is_proxy":true,"latency_seconds":15508.583,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":9.82},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":262144,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":0.95,"upstream_model_id":"kimi/kimi-k3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"8a36368652d98ddf1638e70d90443a973e6fde1ed30c23864ed240576fbf0785","result":{"confidence_high":68.19120000000001,"confidence_low":29.696800000000003,"sample_count":null,"score":48.944,"score_display":"48.9","source_stated_rank":24},"run_fingerprint":"65c11d5973c35e8572477a59b9eab7bf6f51735983e35f1fb70d3b17e966ed30","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ca07bc6e7f4863be65708224b1ad0a7cb79f949f1101e44e638f43a188dca3d3","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.103719,"date_is_proxy":true,"latency_seconds":200.263,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.971},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":262144,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"max","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"kimi/kimi-k3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"aefce118a09c2e61c4298eca9d33c4e29ca79caf14ac5cede8b46b9949916002","result":{"confidence_high":89.08816,"confidence_low":85.28184,"sample_count":null,"score":87.185,"score_display":"87.2","source_stated_rank":18},"run_fingerprint":"9663ffde3615991211004cd57ad9959cdbbda8ba4a72d30f9e9b9bf92764ae57","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"764dd776a5e6bcfa7b0cddd91976b40b8e66446554214924e5e855b6efff6802","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (max)","source_effort":null,"source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"be4a8ebda0a52db608c03f8b63fac7085846a309d5487647040f6af805424b4e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.4907407407407,"score_display":"59.5","source_stated_rank":10},"run_fingerprint":"afc8c3ce63e6d275381910516eedd6880148e1f77603a94b9444c73804530c98","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d05bdb1d75f8b979af2002ab0f103a22c0b9a7f4630ded33311bdb44fbc46b90","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (low)","source_effort":null,"source_model_version":"kimi-k3_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1efdb6b973cf2e3bf2d6839b6ddd11c3b0f92df22d12dcd75c77e0afa6a2777b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.1574074074074,"score_display":"51.2","source_stated_rank":80},"run_fingerprint":"768452d42cf123f0b090811d6eb7045db15ee641df53a703e85a10a87a182ae3","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3b86ee4803d1cc4bcf7da32dc66540f9ccccb21f091d7a80025a2a0145793d49","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3","source_effort":null,"source_model_version":"kimi-k3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"3fdb9dcde4db7a508896d78801b5d677754cf8db0db2d57f43540dec8101962c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.6805555555556,"score_display":"58.7","source_stated_rank":17},"run_fingerprint":"850595bfe310931ad55d69dfb56e3276a7769ff018ee4dfc91d5370fcfcf16f7","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_06de9b035fc59d806e823733","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b33cb281bbb64f849b6f148f8d2b95fc36316d774966734a878150e91610feb1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Max) · scicode"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3","upstream_model_label":"Kimi K3 (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"4a84b556fa5c68bb29546c1dfe71b9a2d8d15d2591c5167b02dffff24b038327","result":{"confidence_high":64.999478,"confidence_low":53.732145,"sample_count":288,"score":59.490741,"score_display":"59.5","source_stated_rank":null},"run_fingerprint":"d1f507013ec9a735e647207c8535425e735c376dd2425e1b7eba82f1f3bd1558","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0a3a4182a09c1cacbe6ba6fd869e35632f6eef4ab20a7e0a0d76dd929140efda","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Low) · scicode"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3-low","upstream_model_label":"Kimi K3 (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"df675f46f1d59b10d47251fbbfc133520d9b320d5184b34b17eb6dcc29365b5d","result":{"confidence_high":58.355539,"confidence_low":46.898452,"sample_count":288,"score":52.662037,"score_display":"52.7","source_stated_rank":null},"run_fingerprint":"4e443239654d152d46e918078cc2db49967fab71c010e025106ea86c89dec1ac","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"91aecd0f3cf3b86ab9b8124bee3f1e69f7a0157cea54606b26eac49e11bd2c1c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"1.281554","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"kimi-k3-max","source_effort":null,"source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"26f8e2ed3572372625e53bc30a10bd0c1ddf914250ae5e5186f642f3fcd07773","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1524.47,"score_display":"1524.47","source_stated_rank":17},"run_fingerprint":"b4b7473e1be5abd696bfb9d7cbd900a8a171007d541913ee536c20813dfd97b4","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"163ee0251c5826c996973fdf5de8cb5d07a35ca26ee9ca5b02a6577f4830005e","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":10.007576,"date_is_proxy":true,"latency_seconds":999.021,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.746},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":262144,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"kimi/kimi-k3"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"6348db9d40897cd3cee5e1b77e05108f46063641b074397184c95b58ccee7a39","result":{"confidence_high":90.35016,"confidence_low":79.58584,"sample_count":null,"score":84.968,"score_display":"85.0","source_stated_rank":13},"run_fingerprint":"a777402a173ff43b689e13f7be4eb07162fef60a087f4c69f7fb4f8a7999e51d","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":6,"at":"2026-09-29","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:27Z","last_confirmed_at":"2026-10-02T19:14:23Z","observed_at":"2026-10-02T19:14:23Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7bb74eaf8fe19753f881acfc91ceb7a76c45b4aa3a88af86812f0c5d5b004c17","metric_details":{"benchmark_page_updated":"2026-09-29","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":17.590352,"date_is_proxy":true,"latency_seconds":5202.941,"result_evidence_match":null,"result_evidence_url":null,"source_listing":{"last_listed_at":"2026-10-02T19:14:23Z","source_confirmed_at":"2026-10-03T04:34:59Z","status":"delisted-retained"},"source_locator":"Results · Overall accuracy","stderr":2.69},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"06843bb07091eafe9bc40a5d49e32ed406fc24be8bf8da8481285130e6d0b9a5","max_output_tokens":262144,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"06843bb07091eafe9bc40a5d49e32ed406fc24be8bf8da8481285130e6d0b9a5","temperature":1,"top_p":null,"upstream_model_id":"kimi/kimi-k3"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"9b4ed088a5ca764e93a2625ba6a2027c3cc8147d64e6443d26fb46a3ec9faf8e","result":{"confidence_high":90.2354,"confidence_low":79.69059999999999,"sample_count":null,"score":84.963,"score_display":"85.0","source_stated_rank":13},"run_fingerprint":"1e270f3b24150805b844b2045ac57c53e30877aab3518273f2ab44cb9ce6b654","source":{"content_hash":"5a4f187719aa9f1090eccde98232aeb685e690c5d5a52acb9a08901409c5a9e6","fetched_at":"2026-10-02T19:14:23Z","first_fetched_at":"2026-10-01T22:31:27Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-02T19:14:23Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b8823b1aab4afda1a9f03f49","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"38eddb64fd750121c957bebeec0538e411086dd581b417c02704bec33a57bd3d","metric_details":{"license":"Kimi K3 license","variance":10.564211160359156},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"kimi-k3-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"12d56964e70f85acee4e4407a923b58416e62958eb582ac2710761643c8d5951","result":{"confidence_high":1664.123602159542,"confidence_low":1651.382804359461,"sample_count":16513,"score":1657.7532032595016,"score_display":"1658","source_stated_rank":13},"run_fingerprint":"ac26d26e5d7e5680ffc7043eb504259b5dd60a668ce2d6c4511d0fcd800488f2","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"65c61144d299e0268b60206d057ae772be8bb0944371ec5dd6864210f3ad6592","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"537407487fdbd65432183b4d8ff02a5792f20e16050bd7c3d36f7718c11f92a2","result":{"confidence_high":1685.47,"confidence_low":1663.05,"sample_count":4547,"score":1674.26,"score_display":"1674.26","source_stated_rank":9},"run_fingerprint":"e8e62bbd5e7828b688a4621b90b707c82c00e01abdeff9f195c9753b3dfb9285","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c2007048353487e03e64ff8c5dee366b3fe723f45de299255025e2337119c871","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"8def5ce4f9c29d1ceb1fbe271d3ab21fe88d2533a452a1ddabbfb9d9427de49d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.57,"score_display":"82.6","source_stated_rank":17},"run_fingerprint":"4915ad781ae3efb00143091adb61098702b847cc4f3837b906ff1d4cba447949","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"afa80cccc10beed5873f8e0a730703c33211f1d2a276617f45085d0fbbf13c13","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.341034,"date_is_proxy":true,"latency_seconds":608.575,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.649},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":262144,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"kimi/kimi-k3"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"8075a5f4efa7ae6cb6bc82cde17f422d1c721e9d0914e0b4a111adfb110a1b84","result":{"confidence_high":82.17104,"confidence_low":79.62696,"sample_count":null,"score":80.899,"score_display":"80.9","source_stated_rank":9},"run_fingerprint":"22c0054b77cb0b57f140b1f6d4c06248dac834dd0c1c4d1723746a5c24fddb55","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3a3b2af30725252e205c7697","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5c8a0f6667e58d2c6f817b9b788009ae886461265bf92a9f2727e6863f0eade5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Kimi K3 (max) · terminalbenchV21"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"kimi-k3","upstream_model_label":"Kimi K3 (max)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"271baa990e04773de86777cbb53557539ec48636df9c0abd064fd07f6d4685e3","result":{"confidence_high":90.97258,"confidence_low":76.166864,"sample_count":89,"score":85.018727,"score_display":"85.0","source_stated_rank":null},"run_fingerprint":"0640cf31ac8a050d43392a2807f118eb18172051ec94f8b6ed9280e236e3bd9c","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (max) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5a6dab8e5feb2a5dc6abcfcaccf9ea521a96ac2402cfb495c5dd70973c2f05a1","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Kimi K3 (low) · terminalbenchV21"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"kimi-k3-low","upstream_model_label":"Kimi K3 (low)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"9da6406d6f042fc2ec4b4e9c201ef65fcf37f79286e1a72f7b15fe9ef9b7f7dd","result":{"confidence_high":88.91861,"confidence_low":73.194352,"sample_count":89,"score":82.397004,"score_display":"82.4","source_stated_rank":null},"run_fingerprint":"9ddc22e815482cc15bfe9450660132b95c056622b9dd2aff5fbc0a39e3928b7c","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (low) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_2aad6f13383503ddfc35685a","split_or_window":"kimi-code-harness","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":70,"at":"2026-07-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:18Z","last_confirmed_at":"2026-10-05T12:33:36Z","observed_at":"2026-10-05T12:33:36Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-27","status":"fresh"},"measurement_id":"4505e4fea1f7e8e76708a887e917f6bbf72d1e1c430657734afb33e4ebb808e0","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · Evaluation Results · Terminal-Bench 2.1 and agentic-task footnote"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Moonshot AI","evidence_verified":true,"freshness_proxy":true,"protocol":"max effort; temperature 1.0; top_p 1.0; Kimi Code harness","reasoning_effort":"max","result_published_at":"2026-07-27","source_content_hash_method":"raw_bytes","source_content_sha256":"57de265b5842dfa465c6e73b368b0e15a89b8793b5450528dad577da202cc6fe","source_published_at":"2026-07-27","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Kimi Code","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"3f3f480a640e9af2c712d5f5d4f3f18daae89f448e3c5488882876029e1bbd72","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.3,"score_display":"88.3","source_stated_rank":null},"run_fingerprint":"42364a47f7bba93f674c1452053db808dfc96de7734e4408e7125dbcac91ace0","source":{"content_hash":"a4cfbbf5b6a39cd0579b65c1b7fc3bfa4f21bd4f4183b560ba850a8a6ccdfca7","fetched_at":"2026-10-05T12:33:36Z","first_fetched_at":"2026-09-03T09:21:21Z","homepage_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md","id":"moonshot-kimi-k3-card","last_fetched_at":"2026-10-05T12:33:36Z","license":"Kimi K3 License; minimal factual score fields with attribution","locator":"Immutable README · Evaluation Results · Terminal-Bench 2.1 and agentic-task footnote","name":"Moonshot AI · Kimi K3 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/moonshotai/Kimi-K3/blob/main/README.md"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":73,"at":"2026-07-24","basis":"evaluation_at","evaluated_at":"2026-07-24","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bf38e7c74d030102b5dfe8fa25dc6aaeb9c7038cd18a31ba1c63cae7cc4414be","metric_details":{"cost":null,"pass_2":25.257731958762886,"pass_3":20.103092783505154,"pass_4":17.525773195876287},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"kimi-k3_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"d4add4b3535a169ff2af2158c9e2174929b38e4bf3719273a4645375b4d4410c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.11340206185567,"score_display":"37.1","source_stated_rank":null},"run_fingerprint":"69e72785d50eba470254dfe111f42a67ed6d6af76229f58eccdde9fea3564a6a","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e504c9d67d1c36605c13c363bb3295609f4fc6c0f4e0dabc1402f903e4fd60a4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Max) · tauBanking"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3","upstream_model_label":"Kimi K3 (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"900f1cebfbd004696d5a8050f6d74f9507066d10c6e1fa5f5d42247592340fde","result":{"confidence_high":55.861171,"confidence_low":36.403925,"sample_count":97,"score":45.979381,"score_display":"46.0","source_stated_rank":null},"run_fingerprint":"07c6a56c83381a75c0859761406f6a0c1a97a3d1371204ca6bcbc930b5b400f8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Max) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4b82a48565f569b525181f7164b96b206efe49cdef9fd5581ff56006bfe7d542","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","source_locator":"Leaderboard models table · Kimi K3 (Low) · tauBanking"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k3-low","upstream_model_label":"Kimi K3 (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"75ce50e7046f50f65f59c357be051340b595491b65d08c88dad0820da27b89c1","result":{"confidence_high":51.594814,"confidence_low":32.340388,"sample_count":97,"score":41.649485,"score_display":"41.6","source_stated_rank":null},"run_fingerprint":"0c23d08a0465c238d29f76503b79fad10f7e2ef17eb4da37f35d18fc5693f16c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Low) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":76,"at":"2026-07-20T17:00:18Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-20T17:00:18Z","status":"fresh"},"measurement_id":"4f073ccd2feaa113ff4837fd15be5ef372b7ca3eac24411e1390d4c1b884501e","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.35,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=kimi-k3 (max); createdAt=2026-07-20T17:00:18Z","source_row_created_at":"2026-07-20T17:00:18Z","task_pass_coverage_threshold_percent":75},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"kimi-k3 (max)","source_reasoning_label":"max","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"042605a717c9dd1da7ca1d947b7eea14102dec6acfc2831f28a67f6864b42857","result":{"confidence_high":84.64999999999999,"confidence_low":79.95,"sample_count":1000,"score":82.3,"score_display":"82.3","source_stated_rank":2},"run_fingerprint":"75155156ad4d43db1e5c03aaa45c4b4d4ce42fae26f0bade7bd9df0562b0b990","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=kimi-k3 (max); createdAt=2026-07-20T17:00:18Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0807b67ce21180575318002a8ceb750d8ff3ada8d64ec8990af848e26b3a5ba4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.64","mean_standard_error":"4.3","model_release_date":"2026-07-16","notes":null,"standard_error_points":509.99999999999994,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3","source_effort":null,"source_model_version":"kimi-k3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"e2521760eef811b2e68a23e12bbc2163c62b60f70494f9554a326f5fc036e763","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":50.6,"score_display":"50.6","source_stated_rank":23},"run_fingerprint":"cf53cef99bfd77e62aa0f8157bab838ea65395dbb1aa62f77c94f5f5f25b250c","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9b48cd9b9e58065d050a20a9","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3c63826a520c9e2c1c6c9984d0e492b22d5266a2d779f8c5c266a7645cfcce92","metric_details":{"confidence_level":0.95,"license":"Kimi K3 license","session_count":132505},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Kimi K3 (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8da31e5bb38ab3f8325dacced47d48a13140e35b16c98b0ea4428f75abaf89f2","result":{"confidence_high":0.04720801040120649,"confidence_low":0.03647210983229049,"sample_count":12305667,"score":0.04184006011674849,"score_display":"0.0418","source_stated_rank":14},"run_fingerprint":"c59a0896be8ec897d456a36758a6ddb91b430cdda28d0daeefab825a00df1d82","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"eda27db1d8563c494f3bc7439068c4764c459071e4c8de1e4197b55d5f9328d7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-16","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K3 (Moonshot)","source_effort":null,"source_model_version":"kimi-k3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"4773551f57125bed55432f6478c79beaea73730ddb3a7beda378f762751e4d9d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5165.035000000001,"score_display":"5165.04","source_stated_rank":25},"run_fingerprint":"351c2df3e80fd46dbbb9ffdab31b08507f803cf10204b7ea851ae64e3bb239fa","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_97cb45dbe3d881ac6db37643","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"05831d136e981f5fd1e6760bbd5fd1feb4340fec253a73c72a6077127661acbd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","reported_confidence_interval":"-20 / +20","source_locator":"Leaderboard models table · Kimi K3 (Max) · gdpval"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"kimi-k3","upstream_model_label":"Kimi K3 (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"1aa72bed35428345df457904c86f4c0431461aa30f643fdae4ac2d4a5ff82c65","result":{"confidence_high":1559.96,"confidence_low":1520.01,"sample_count":220,"score":1539.99,"score_display":"1540","source_stated_rank":null},"run_fingerprint":"bccb54694d373f51e1613575b1d6bb95691e7bef3ed96ae1106f077a8d9f7f50","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_2de1840141225fae3ff55b20","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ab38cac3a3f52fc2e76db6523f27767e9d70f12ea1982251db39c8540aed7381","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1048576,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-16","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Kimi K3 (Low) · gdpval"},"model":{"id":"moonshot/kimi-k3","name":"Kimi K3","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"kimi-k3-low","upstream_model_label":"Kimi K3 (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"96be066dc5c5c5df800858de8a7a75bf96bc0c386689ac7907f83388cfa42b7f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1118.6,"score_display":"1119","source_stated_rank":null},"run_fingerprint":"f21bbca31475f33bb87597ace765c73bb90d695c94c3a4112178d6558ee2afd2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K3 (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_841aba49995f4b40bc9b569c","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"29cc0d0bc3494c57a4f6c2c9e9d150e33125648eb77008bdc540ac17d1b50c2e","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-08-13"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 3.7 Flash; model release: 2026-08-13","source_accessibility":"API access","source_model_name":"Gemini 3.7 Flash","source_model_release_date":"2026-08-13","source_model_versions":["gemini-3.7-flash_high","gemini-3.7-flash_medium","gemini-3.7-flash_unknown"],"source_reasoning_efforts":["medium","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"94bfe1ef447042f33e4f90c226629de4636a815fce74e8af0030573cd40d2934","result":{"confidence_high":160.06,"confidence_low":155.41,"sample_count":null,"score":157.44,"score_display":"157.44","source_stated_rank":14},"run_fingerprint":"2e1f061d78823cbdc5ba2f3911549e4ef570a92893e2c0ab5340e7619ea34725","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2c8a392b509c1c0521fc9bf3","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"453eba479b7f67daba6baf727800156f8c67f4c1d19d360840e3d9c7cf30527e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (high) · intelligenceIndex"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash","upstream_model_label":"Gemini 3.7 Flash (high)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"83f814e1c3d17e8312213f668f559c12cfbc5c5e6a62f37309b29ae5c25850a1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.429532,"score_display":"39.4","source_stated_rank":null},"run_fingerprint":"23300dc60213f8f83b1f0063e6f1b2e1dab4ff366b7892b443d725c03c3bd2dc","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (high) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ad84e8c721689b2de573e669c3ee022feaafec94267bb9964d9e2e6b65f26896","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · intelligenceIndex"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-medium","upstream_model_label":"Gemini 3.7 Flash (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"1a248924ee083b004c565b566f335a642942c2dbc6721e6751de4fc8c36ba958","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.617752,"score_display":"39.6","source_stated_rank":null},"run_fingerprint":"c057575b5413dfb200002948cd31f0f5a02a6046d4497a7db162f645bb4561fa","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"45b5a6a3ac02af7f09325e434987a9b952020ec1abc0fccf2f39d07bdd3004a5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (low) · intelligenceIndex"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-low","upstream_model_label":"Gemini 3.7 Flash (low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"cc400fbb1c4daf6c5ca82c89444d618b6568d9733a112239ea10044fcfd00bcb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":36.945911,"score_display":"36.9","source_stated_rank":null},"run_fingerprint":"cc5a2e932e9fb71b84b667e208e8b0af42eb5c87a34aa2358650ff494769eeef","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f4321bbd17f9bd2f2b0f0c2715413aa60232dd74d23a1d7a202d2b314bfbfc36","metric_details":{"license":"Proprietary","variance":6.968441496646932},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1492.8510050863104,"confidence_low":1482.503255368853,"sample_count":21539,"score":1487.6771302275815,"score_display":"1488","source_stated_rank":17},"run_fingerprint":"62f408bbf69910b930e417c3a873b2c19ba2f16010f589ff450e075aba176550","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"835f704d338d41281ab490b91dcb20ae5c13b336a95df7608dd93621ede8a7d2","metric_details":{"category_scores":{"Agentic Coding":58.282999999999994,"Coding":78.8885,"Data Analysis":67.96433333333333,"IF":79.92525,"Language":85.455,"Mathematics":93.46799999999999,"Reasoning":87.798}},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.8260119047619,"score_display":"78.83","source_stated_rank":16},"run_fingerprint":"9157b56ec02d65ac7dedcd65ea797c279f4277af0b25b1aa2ece8b761b8bd597","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a6a08d25935df2e9b1b2892d46586b9249286c4c6c815c63df3be896d5d73d71","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (high) · hle"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash","upstream_model_label":"Gemini 3.7 Flash (high)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ca7773258dfcd0ce156963016827db6b1137cdd4c4f0777571ce92f9798bf5f5","result":{"confidence_high":49.977998,"confidence_low":45.766372,"sample_count":2158,"score":47.868397,"score_display":"47.9","source_stated_rank":null},"run_fingerprint":"1c45f03a59a55567820a001236da0986e05bd202fb01bcadbcdcc34dd8527c6b","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (high) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2847b79d20858da530df66e670863c759753fd6b557edd77ccc219133e9f2610","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · hle"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-medium","upstream_model_label":"Gemini 3.7 Flash (medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"10300bc284bc973ba60f32222f4abf24fa71678f23c1a3d72b17f9af0106d305","result":{"confidence_high":41.046774,"confidence_low":36.934962,"sample_count":2158,"score":38.97127,"score_display":"39.0","source_stated_rank":null},"run_fingerprint":"7a7aca6a234e6b378db74acbe90c14f601198b9b1f92779022c3396083341b30","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"32a0c1849e60738316a9650d7cfbc5cffa8039517ccec04f4a9b334503dbf42f","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (low) · hle"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-low","upstream_model_label":"Gemini 3.7 Flash (low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f6976a0000887752e875991f962690972cfcbc1e8cc959b12f5a2624a32e0926","result":{"confidence_high":37.164016,"confidence_low":33.139082,"sample_count":2158,"score":35.125116,"score_display":"35.1","source_stated_rank":null},"run_fingerprint":"9f5ec4eed0bca12e877d10fa95fe2a9cc6eeaf56d532d73d1bc1ebb042d8e2d6","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:30:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:30:25.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4696322a64733ba1048671a9f6db9220e1f360a5a16e6001ca72fddb1aea773d","metric_details":{"best_score_across_scorers":"0.692","model_release_date":"2026-08-13","stderr":1.460648312734278},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"84ARuGyhQsjgh8umRmBvas","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F84ARuGyhQsjgh8umRmBvas.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/84ARuGyhQsjgh8umRmBvas.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"9f5799cd60e18ca9aa0eecd6743a82f7e7c58c7b690fd09e185585ab33e03352","result":{"confidence_high":72.06287069295918,"confidence_low":66.3371293070408,"sample_count":1000,"score":69.19999999999999,"score_display":"69.2","source_stated_rank":9},"run_fingerprint":"237f1a849a1439d95bbb23d41ce4e32454256aaa4e7b407cabb8a56042ad1ee5","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":39,"at":"2026-08-26T21:27:54.27784Z","basis":"evaluation_at","evaluated_at":"2026-08-26T21:27:54.27784Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a1f891efafce633d2e1e1c9fd322c51e656c7265cdc8f54fc0674e7f2f7547c2","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.7-flash","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"eedb1f0afd9f6085b8b6ca56ec8e3d396bc2bbf1f7694953cdb10490c1ff4b44","result":{"confidence_high":76.16448160267086,"confidence_low":70.22106056600384,"sample_count":null,"score":73.19277108433735,"score_display":"73.2","source_stated_rank":4},"run_fingerprint":"9594e3ac743cc0114c05ae37e67b20e1291db9c6b72fbedda69de20328e1051d","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0277cd323f82ee16a210fc5a","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c18bbc0350988a0b12768cc3ceea766d2c3656cf17abb93e61905b456cc2959f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.023122,"date_is_proxy":true,"latency_seconds":8.773,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.488},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":95.455,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.7-flash","zero_shot_accuracy":92.424},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"1fac15ea3e046e93cb7454fa4c1e2ee16467a700afdd3ae905c4362c16337d7c","result":{"confidence_high":96.85648,"confidence_low":91.02351999999999,"sample_count":396,"score":93.94,"score_display":"93.94","source_stated_rank":5},"run_fingerprint":"fed0d0cc6024079ea84f1505cf4be6a9f3b8982c584aa959a7f4d691ef297142","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4a1544a95b871e4241599d986746641a67a8e0c13465bc471a4da6f05067a763","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (high) · gpqa"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash","upstream_model_label":"Gemini 3.7 Flash (high)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ac0dcab6b74a910121200f92aaa348238a16bbfb66523187d8c3ed215ac0adfd","result":{"confidence_high":96.943251,"confidence_low":90.452015,"sample_count":198,"score":94.545455,"score_display":"94.5","source_stated_rank":null},"run_fingerprint":"dbb7a657caf38d1a31a03bd0c67a015fc5dd47f54c60d4327518fdacc1af44a3","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (high) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b4dc0c10df3c098637c73249af51eea8356313adf33eac27c02a04282ceebee6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · gpqa"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-medium","upstream_model_label":"Gemini 3.7 Flash (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0a694ec4ee476db37f502ac1ea300435da973d6f273be871680644b1a0893b31","result":{"confidence_high":95.12173,"confidence_low":87.517329,"sample_count":198,"score":92.121212,"score_display":"92.1","source_stated_rank":null},"run_fingerprint":"48f0341c3f1dad808b12f399e5b8afecfdb334e14b9a0c59e7aba008b3207ee6","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"57c832a33748e255f63485cc031ff4f50c196d4b540714ff4a72fc6f41c98221","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (low) · gpqa"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-low","upstream_model_label":"Gemini 3.7 Flash (low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7bb22721ca9375d0d15a0a66530e1e29639d12eb0d811862a144e7faf3757e0a","result":{"confidence_high":93.528005,"confidence_low":85.147551,"sample_count":198,"score":90.10101,"score_display":"90.1","source_stated_rank":null},"run_fingerprint":"4d706f2cd959b5aaecbdd38dc78288b6b484df25659bca7182a1309b5e9fe4e5","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":51,"at":"2026-08-14T18:29:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T18:29:12.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2f3b8ae8f0b51da32d18a92f6147afe362265516581bb2ffbdc663a9a573a498","metric_details":{"best_score_across_scorers":"0.9482323232323232","model_release_date":"2026-08-13","stderr":1.345652529224626},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"XyVL3VffJ6UboY97vxpohL","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"dff3e8f5b530f85eec01da5a26f37e85aa74007e516cbbb8645af25795d18c1b","result":{"confidence_high":97.46071128051258,"confidence_low":92.18575336595205,"sample_count":null,"score":94.82323232323232,"score_display":"94.8","source_stated_rank":5},"run_fingerprint":"6288d871c78d8a73f2c8aebb63374570fd5c33bfa91c9823452e36bfc5e0acac","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":39,"at":"2026-08-26T18:47:39.384499Z","basis":"evaluation_at","evaluated_at":"2026-08-26T18:47:39.384499Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c0f3cd4572ec616474ea8ccd29095f0c90ee9459f5ce3db4776108cdfc281bea","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.7-flash","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6410c04ca8d9358fbaff9676ea335779441154ab8dc7d4b117a06750da40100a","result":{"confidence_high":93.68284596938167,"confidence_low":85.10503281849711,"sample_count":null,"score":89.39393939393939,"score_display":"89.4","source_stated_rank":9},"run_fingerprint":"16500bf1408c7c05b3fcc4eb29a136ff300bb8416f11a987e65bde52b571267f","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_062fc3bd8993c44fd1ee46ec","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"239599b01e76dd16f85572d0f752ae9d4074dfb416f13b57e5759d320d0d34e5","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.010817,"date_is_proxy":true,"latency_seconds":4.159,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.296},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65536,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.7-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a983b691b9113334ce197b6ca9115fc79138e1734adb8a3434cea76f60af731e","result":{"confidence_high":90.70116,"confidence_low":89.54083999999999,"sample_count":null,"score":90.121,"score_display":"90.1","source_stated_rank":6},"run_fingerprint":"cfd3a2efaa9e58d7029c8cb7a3f6ca9e8ac33df5eb1c769f4d603afe702160e9","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":51,"at":"2026-08-14T18:29:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T18:29:12.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"156398fe41854f83b8a011fadfcbe8b05703643c1dde170b3a3eed51041cacc7","metric_details":{"best_score_across_scorers":"0.9722222222222222","model_release_date":"2026-08-13","stderr":1.8586784946171873},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"kQjrYCaa7yYyMXofeR23Dd","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"32f8df7bfa32c631c95c02add8808505ef388aa6348f5af1da1f0fd50cb3c08f","result":{"confidence_high":100.0,"confidence_low":93.57921237277253,"sample_count":45,"score":97.22222222222221,"score_display":"97.2","source_stated_rank":31},"run_fingerprint":"a44a6a3f11068331b0144fdb60cbfe35d2ee37b4c6567fc92e9b3a303a1f8fd8","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":51,"at":"2026-08-14T19:59:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T19:59:58.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f04ed336f73b50245211238ea7caf8c9742369790f9868f65d72669719bb15cf","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.7157894736842105","leaderboard_private_problem_count":285,"model_release_date":"2026-08-13","public_example_count":10,"stderr":2.6764156649364654},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"axMdgNDC8UYYLnZYLHZpF5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaxMdgNDC8UYYLnZYLHZpF5.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/axMdgNDC8UYYLnZYLHZpF5.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.7 Flash (high)","thinking":null,"upstream_model_id":"gemini-3.7-flash_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"5c4ffbaa4694435d16724924bd3e92e829ab6da23f61d6ca87985f02225a6a87","result":{"confidence_high":76.82472207169653,"confidence_low":66.33317266514558,"sample_count":285,"score":71.57894736842105,"score_display":"71.6","source_stated_rank":23},"run_fingerprint":"7cae70d1d7feee1906ae244aa8c95c648293d66d200ee5cc6b4b3dc9e45e7550","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9fd8ca5d947e2e487d024c8d9dc257d8effb5b7f6d11a853d8c6f7776e9af00c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.24944951875000002,"model_release_date":"2026-08-13T00:00:00.000Z","results_url":"https://arcprize.org/results/google-gemini-3-7-flash"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"google-gemini-3-7-flash","model_type":"CoT","upstream_model_id":"google-gemini-3-7-flash-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f6c9bfb4f2fb48d9291daf5ef9fc0e68135afbc6971eff8c25aca2f196043cbe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.58333333333333,"score_display":"85","source_stated_rank":30},"run_fingerprint":"5b1439acf75cc1ac28222902c0b7f74ede5f6139c9c5e32cac7d26868be447ae","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6ce992eadd7bc9bf53c8c885c9fff79bd3b348a20deff711b389778aaf79bec7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.11582842499999997,"model_release_date":"2026-08-13T00:00:00.000Z","results_url":"https://arcprize.org/results/google-gemini-3-7-flash"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"google-gemini-3-7-flash","model_type":"CoT","upstream_model_id":"google-gemini-3-7-flash-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"09094eb49555dc37ab690646d58b514df115fb02be2f4e235807824fd816cbd3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.74999999999999,"score_display":"64","source_stated_rank":69},"run_fingerprint":"46e1220ba21a49fe73e49db81b172dec51f5160fed3f5759e1a5126de66b9ea0","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bcc743169bb5f7d8602524fca7430db9ccf0040ca1bed156656384297fe6d3b1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.07938655,"model_release_date":"2026-08-13T00:00:00.000Z","results_url":"https://arcprize.org/results/google-gemini-3-7-flash"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"google-gemini-3-7-flash","model_type":"CoT","upstream_model_id":"google-gemini-3-7-flash-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bde06d9acc2d28e48f40d93509dd397892fd03d3d8b562ca05dde1b70d584a81","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.916666666666664,"score_display":"53","source_stated_rank":91},"run_fingerprint":"52c4f156baffcec54e007643fa32166c6791416be869d3ba19dbadf7d647b269","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"40d7cd6e7487babe9fad24481ba586660e8a45328bbf31d271f01b9a63751af4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (High)","source_effort":null,"source_model_version":"gemini-3.7-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3d61661baf2e1783105e9cb0d6a6df2e61c537cb79e2c430a65f0d073a80c95e","result":{"confidence_high":87.946654,"confidence_low":80.48972,"sample_count":360,"score":84.58333333333333,"score_display":"84.6","source_stated_rank":31},"run_fingerprint":"d53700c7d852b9cc7a69dbbe24dd5bb1a34e2d3e949660c4004a4afe9e63c853","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e0ed6ffdb1ceb629584b9cbe025d8776bdb2417a97f4a3e4ca6fa004de17c681","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (Medium)","source_effort":null,"source_model_version":"gemini-3.7-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"09125635b3a701b9e7f4b7aef7e7249a71a7b3aee5cb2c265e98f7910162ec43","result":{"confidence_high":68.54658,"confidence_low":58.663063,"sample_count":360,"score":63.74999999999999,"score_display":"63.7","source_stated_rank":72},"run_fingerprint":"69f3905c25e891a53f559b8b492326856dbc7d01487ad944bbb664093a4ea9be","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"452464e9b8875abf991ee7d8f0ae092afaef5eb89722f23a02407efd57288861","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (Low)","source_effort":null,"source_model_version":"gemini-3.7-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5da0c33752bc0429b864c9a7de5588ae973d24cf921060c27ea899bff58db415","result":{"confidence_high":58.014929,"confidence_low":47.756814,"sample_count":360,"score":52.916666666666664,"score_display":"52.9","source_stated_rank":92},"run_fingerprint":"21bb5b086758516bce109a5a97ade572b86fb8af48af82cf8eb77da886f39970","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":51,"at":"2026-08-14T19:59:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T19:59:58.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"38f30d5e5f57cacc65d1ccf06f2809e2eec483fc408d8a55fcd3c8e8987cb917","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.36585365853658536","leaderboard_private_problem_count":41,"model_release_date":"2026-08-13","public_example_count":2,"stderr":7.6158512175590225},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"KptKXK5nE2Ns2WbiaSSnbc","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKptKXK5nE2Ns2WbiaSSnbc.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/KptKXK5nE2Ns2WbiaSSnbc.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.7 Flash (high)","thinking":null,"upstream_model_id":"gemini-3.7-flash_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"4c066fcbc43cca1fab40e9bcc635e96bfab7551d349b999e087229dcbb8a284a","result":{"confidence_high":51.51243424007422,"confidence_low":21.658297467242853,"sample_count":41,"score":36.58536585365854,"score_display":"36.6","source_stated_rank":28},"run_fingerprint":"82e46eec4440155377831a4ad3dd2f53cb6777eb3ab0029b088f145500b26314","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cbe6e9647cced364fa3eff0806d335aed7bd26f8a1cebbd406eb8cde73472a4b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.014052,"date_is_proxy":true,"latency_seconds":6.283,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.753},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65536,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.7-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ad637647bfecd7d8373ecf4a47593ea8e5caf0510244dc11a31e9c01d546ed47","result":{"confidence_high":90.43588,"confidence_low":87.48411999999999,"sample_count":null,"score":88.96,"score_display":"89.0","source_stated_rank":5},"run_fingerprint":"1039a5e4684dab617d6e3c3fb7102d50b709173afea20855f5813ab9d766980b","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"25365c89f9aca3cc425f49df2c9243792cfb8de24b6ecc912e07cc498bf46fe5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (high) · mmmuPro"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash","upstream_model_label":"Gemini 3.7 Flash (high)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"61a0a6d391f08c14d0050988a269655681b4eddd83e0aef228e76322d965f1b6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.491329,"score_display":"85.5","source_stated_rank":null},"run_fingerprint":"2aa820c7bafcd783a6ef9e52d8410b3fea4584800914ba066a275e0f084a74b7","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (high) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6b2637312289299f4519f16eec3cd14cf3958a3fc52436e98218e98123ccdd9d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · mmmuPro"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-medium","upstream_model_label":"Gemini 3.7 Flash (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1143677793a35ac7e990b920460f59c744571b03cd5669d19ebee06b294021de","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.739884,"score_display":"84.7","source_stated_rank":null},"run_fingerprint":"cc91735a3c806d1dc4d14fed27f4c69c900ca54085b444f910495a115a8e5597","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"174c61fc965dc31c81a8ed28df5892004c5daad4c2ccbe3ace78466bc64b61cb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (low) · mmmuPro"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-low","upstream_model_label":"Gemini 3.7 Flash (low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"78e1c37fbb09aeb4875e9cd5d486c43a7d9b8c6f3d6ae8b66a3a92b2823644bc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.855491,"score_display":"84.9","source_stated_rank":null},"run_fingerprint":"c981cf80effde1c4d2378140047484be6c4d039d8252a4e91b235cbf02c96a25","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b41d7acb67c72aafd2782ed37cceaf37a2a83bd12e56a6df8ccdf720e7b4eecc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (high)","source_effort":null,"source_model_version":"gemini-3.7-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a5bb28459dc6ca9f40d95004b27df14fa7dca5c2a515f6c507793a00ffc6997c","result":{"confidence_high":24.25608,"confidence_low":7.981759,"sample_count":71,"score":14.285714285714299,"score_display":"14.3","source_stated_rank":80},"run_fingerprint":"d52b610d9dc3a6c021b284550757904a847aeee2121103ed3ce0869c645b8d3e","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"98505af66eb7476ca1766550122ff9f3945fd816ea78911953fa6a39f3026498","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (medium)","source_effort":null,"source_model_version":"gemini-3.7-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"48618dbd81cea7208bfa3128eabaf151ab3715f5ad3d38587e67a4fad9a504a8","result":{"confidence_high":18.451582,"confidence_low":4.570603,"sample_count":71,"score":9.42857142857143,"score_display":"9.4","source_stated_rank":95},"run_fingerprint":"dd2c33a7ac818fc8cce7d6f4e8552106593dbcca8bfe904d1e649b12070f05ab","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9b7b288232ddbe39af77746713793d5d93d7e4a342ec04ab70a302d9c3414acd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (low)","source_effort":null,"source_model_version":"gemini-3.7-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"21a58a127910475983742ea648b808396df9d40af2454a63dcff845e1069244d","result":{"confidence_high":13.716555,"confidence_low":2.258365,"sample_count":71,"score":5.71428571428571,"score_display":"5.7","source_stated_rank":106},"run_fingerprint":"a177d8820cb3eb00c331b63c04315f4a3c276c4e4d4ccb5bfacb5228e236329e","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a1da8c5fb5b07759bbdac05abf322e123e80dbe8d33f6f391bb55d52e1a7f0ff","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (high) · critpt"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash","upstream_model_label":"Gemini 3.7 Flash (high)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"fe83994269c63665bc75f2d5250da632d13e401026e2624dd3db6b3da8af5f58","result":{"confidence_high":24.338634,"confidence_low":7.948856,"sample_count":70,"score":14.285714,"score_display":"14.3","source_stated_rank":null},"run_fingerprint":"93563ee93b07325e88bb62372974e5d0207af5b7c2672b7bd50c8708e90408fc","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (high) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8f7adf56fdd179d0fe8f22693fafa778385b6991551ed119452f55a6452b45d6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · critpt"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-medium","upstream_model_label":"Gemini 3.7 Flash (medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c96c45e169f9d7c724d5eda825a9e7bccb99f4f37f3fc7061bbb5902677e2d8f","result":{"confidence_high":18.53088,"confidence_low":4.547709,"sample_count":70,"score":9.428571,"score_display":"9.4","source_stated_rank":null},"run_fingerprint":"04f7245ed66a8413d83060d7515d58830899ff85e1268896e4f0706630faae3d","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1a7737fcd1d849c6191f3ba7b0609aa32b97f148552257b4834601d6f86542a3","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (low) · critpt"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-low","upstream_model_label":"Gemini 3.7 Flash (low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"0a55c2192402bb7b95ba81f8e9e27eb2f4a70f9dfd3b102fc0ead92284d38261","result":{"confidence_high":13.792148,"confidence_low":2.244342,"sample_count":70,"score":5.714286,"score_display":"5.7","source_stated_rank":null},"run_fingerprint":"ba28b31084673c3dee46c54783035e65ee683ff0a43df0408cb361e058c7f56c","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":51,"at":"2026-08-14T18:29:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T18:29:12.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"487a559fa70ec560e5bd597bf5b485f54a8a2d1944b9d7f023d49e0dbd0936f8","metric_details":{"best_score_across_scorers":"0.47","model_release_date":"2026-08-13","stderr":5.016135580465917},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BnqWATgxsPzSF7ojBEMoG3","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d8794653bef9a795f0ef8224e80dc00951d08a15c7265209d4cbc3fbaf46fd2b","result":{"confidence_high":56.831625737713196,"confidence_low":37.168374262286804,"sample_count":100,"score":47.0,"score_display":"47.0","source_stated_rank":14},"run_fingerprint":"862f83753ce347f7fe3051db30e0ebf1f968abd0ad9df2f07fc87ff4b00859bf","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6a9b123a412548875d0c26477bcc9012c969d15924eb552eb4d64979548331d9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (High)","source_effort":null,"source_model_version":"gemini-3.7-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"69f32bc8259b448a1f8af6930b6b12e08ba568889f8ec89626ba030f9e02c99a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.5,"score_display":"95.5","source_stated_rank":29},"run_fingerprint":"a682cd31ddb5c3b239b27a0cbbcbf03d6714ce8667a23f00ee1f434d78b87c52","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6802a2004a3be105187f4cc1ac5bd089a2107b74d91c71501eb65d9650b37340","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (Medium)","source_effort":null,"source_model_version":"gemini-3.7-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e0283c4f05cecece0081956d619184166de5c21574b103393aa9e22d28d73b86","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.16666666666666,"score_display":"91.2","source_stated_rank":65},"run_fingerprint":"05b5bb63ac899b4ac2ae1b03d3ea7273a82fb1898b138f1f28debfa22c28792a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f125f3463e685d70f8ba689cfe1e4844c47b91b03c329e45f941a7e386a4a2a1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (Low)","source_effort":null,"source_model_version":"gemini-3.7-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"746a4008512722d4136e6d813c354fc12037622b55b34a1b1c93bff26cb38ee3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.16666666666666,"score_display":"85.2","source_stated_rank":104},"run_fingerprint":"002fbe546664b9de61add4154f5f6be41191a0d056e18b14b7d3cc61cae34457","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"afbce0970cd214f7a18a94677a8a70438373edc66fe271c5c1ae370e3c48b807","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.436459,"date_is_proxy":true,"latency_seconds":350.439,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.763},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.7-flash"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"321c692090738e94b819f8bdeec00761108c135f805d6247f2edb8715461ce6a","result":{"confidence_high":84.25547999999999,"confidence_low":77.34452,"sample_count":500,"score":80.8,"score_display":"80.8","source_stated_rank":23},"run_fingerprint":"206c6e18bf6ee66915cd604951d1639f308a277eb251a3a62293df8eb19f9f3b","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"87e9bee21cbd1df04bc424abb64876f913627d00af5fe54fad1e698e57724c92","metric_details":{"ci_attempted":452,"ci_half_points":1.7878625067843175,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":295,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":124.51548672566372,"mean_cache_tokens":16260422.17920354,"mean_cost_usd":2.1762860351216813,"mean_duration_seconds":1244.9565387986725,"mean_input_tokens":16999853.159292035,"mean_output_tokens":107248.30309734514,"median_agent_steps":120.0,"median_cost_usd":1.9044440625,"median_duration_seconds":1064.5330275000001,"median_input_tokens":13967881.5,"median_output_tokens":99819.0,"median_output_tokens_to_pass":101363.0,"median_peak_context_tokens":221180.5,"n_attempted":452,"n_passed":295,"n_tasks_attempted":113,"n_tasks_passed_any":93,"pass_at_4_points":82.30088495575221,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gemini_3_7_flash_high","source_model_version":"gemini-3-7-flash","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"bfefa0da8b857f3d96ac81cd445976b8fee7a53a767175833268ee0896536293","result":{"confidence_high":67.05334923244803,"confidence_low":63.47762421887939,"sample_count":452,"score":65.2654867256637,"score_display":"65.3","source_stated_rank":27},"run_fingerprint":"c2fe90d75526beb903835bbbe2ae4e5582d1bf9abdfa369143d517c93656c2b0","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"0acb853478d86ae02d6b681da40a3076fcbc85c6fea2152b456991cfa9010585","metric_details":{"ci_attempted":452,"ci_half_points":3.086532276415522,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":296,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":117.36946902654867,"mean_cache_tokens":14557599.913716814,"mean_cost_usd":2.0251351826880533,"mean_duration_seconds":1240.376194079646,"mean_input_tokens":15332065.907079646,"mean_output_tokens":93990.8517699115,"median_agent_steps":108.0,"median_cost_usd":1.65422055,"median_duration_seconds":1066.1105834999998,"median_input_tokens":11116142.0,"median_output_tokens":87640.5,"median_output_tokens_to_pass":87132.5,"median_peak_context_tokens":195705.5,"n_attempted":452,"n_passed":296,"n_tasks_attempted":113,"n_tasks_passed_any":94,"pass_at_4_points":83.1858407079646,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gemini_3_7_flash_medium","source_model_version":"gemini-3-7-flash","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"0882774489636bc3a9752122add1f955cf7d5a7a5517c4181eba9f3756c7fefe","result":{"confidence_high":68.57325794013234,"confidence_low":62.40019338730129,"sample_count":452,"score":65.48672566371681,"score_display":"65.5","source_stated_rank":25},"run_fingerprint":"994ac82c716dea01c81e348da0a9b6d644f317ea92827cc4fc044e3ad791916c","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"9738e6d8214a7683b2b782b3c563d4910d8bbeaca102d15f25436423a3dca4c7","metric_details":{"ci_attempted":452,"ci_half_points":2.589649081831869,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":243,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":130.37610619469027,"mean_cache_tokens":14422633.818584071,"mean_cost_usd":1.8322645629424779,"mean_duration_seconds":1252.0731730707964,"mean_input_tokens":15056565.075221239,"mean_output_tokens":73364.9557522124,"median_agent_steps":113.5,"median_cost_usd":1.3913454375,"median_duration_seconds":1102.7230410000002,"median_input_tokens":9612558.5,"median_output_tokens":73220.0,"median_output_tokens_to_pass":75057.0,"median_peak_context_tokens":168146.0,"n_attempted":452,"n_passed":243,"n_tasks_attempted":113,"n_tasks_passed_any":88,"pass_at_4_points":77.87610619469027,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gemini_3_7_flash_low","source_model_version":"gemini-3-7-flash","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"659874875f664508e4c0fc4a62f6ae84ce1fdc2a4c4e26935a2378323c5809e2","result":{"confidence_high":56.35071102873452,"confidence_low":51.171412865070785,"sample_count":452,"score":53.76106194690266,"score_display":"53.8","source_stated_rank":43},"run_fingerprint":"f53dfacf78ba6c513bcac94460050f52c88acadfb1d010c138ce86aea27b4930","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"008102a8047ccdc7fac1ec84bdcb5a3f1900312b86ff7487d1900c92942c8c6a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"124.51548672566372","mean_cost_usd":"2.1762860351216813","mean_output_tokens":"107248.30309734514","model_release_date":"2026-08-13","notes":null,"pass_4":"0.8230088495575221","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-7-flash (high)","source_effort":"high","source_model_version":"gemini-3.7-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"917b3cb5659c0ab3fdad809f590111be69e1d2c271dca6088349f67abd233557","result":{"confidence_high":67.05334923244803,"confidence_low":63.47762421887939,"sample_count":113,"score":65.2654867256637,"score_display":"65.3","source_stated_rank":27},"run_fingerprint":"6eb502acd885edf030daffc4233c4a5198c535a15fa3e36f144ae1b8a9e0277b","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4a6a25b8e79057c30ecf6db32c44979d0c26a893e40740d984ffbedf940f328c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"117.36946902654867","mean_cost_usd":"2.0251351826880533","mean_output_tokens":"93990.8517699115","model_release_date":"2026-08-13","notes":null,"pass_4":"0.831858407079646","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-7-flash (medium)","source_effort":"medium","source_model_version":"gemini-3.7-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"fe419e58a22d69479134b6b07ba5151c6dce1186787fb74ec729a6f944a83e00","result":{"confidence_high":68.57325794013234,"confidence_low":62.400193387301286,"sample_count":113,"score":65.48672566371681,"score_display":"65.5","source_stated_rank":25},"run_fingerprint":"6329d0f50ca5aa928b9395221f034935e48d52a7a0818d73e75741923b73c2cf","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e500826af8bcfe71f26586ace066710b175a9b18079efc880d5043f154f8443f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"130.37610619469027","mean_cost_usd":"1.8322645629424779","mean_output_tokens":"73364.9557522124","model_release_date":"2026-08-13","notes":null,"pass_4":"0.7787610619469026","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-7-flash (low)","source_effort":"low","source_model_version":"gemini-3.7-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"2278d5151e0a9a99aae9e28f3a38056330763226f04117b5799b950e03542d2d","result":{"confidence_high":56.35071102873452,"confidence_low":51.17141286507079,"sample_count":113,"score":53.76106194690266,"score_display":"53.8","source_stated_rank":42},"run_fingerprint":"13279da6cd7504b885926fddb93e99472b04455eaf964ca6032c5f7f6ffa8743","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"22b1e4d71414d87403d69e835e28c010577db272b163d1b82e7d0b77bb72e9c2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.2","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-08-13","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"99","tokens_per_task":"38448","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash High","source_effort":"High","source_model_version":"gemini-3.7-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"e30a0d3670d9c36b1be49ed529b3e9bc98645cc0c8d60332652813f36a69aec8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.6,"score_display":"61.6","source_stated_rank":27},"run_fingerprint":"49753c79198388f4ee76fc620d4c2b846090400f998e8374cbcefd307a363ec3","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"410b78a6891092b79f6dea259d24a517588183d5a92de5037d43ce8616652f69","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.95","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-08-13","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"82","tokens_per_task":"30953","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash Medium","source_effort":"Medium","source_model_version":"gemini-3.7-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"28914c9682145459ac79f155521c94cd34074557e4aa927fc0b3198f7c450c64","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.0,"score_display":"59.0","source_stated_rank":38},"run_fingerprint":"50f2ba47e347e120e5d667ac39545a4895ecaa03bd066f379c3a07d9f3f24840","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"251edd8c9c86c47d639de6d274cc41a07ff8801284e7096d56b1ff741353a0b6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.74","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-08-13","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"68","tokens_per_task":"20594","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash Low","source_effort":"Low","source_model_version":"gemini-3.7-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"97c7b69f2d9e32794e19c4e50fa37121e0c902768d77802ee5159da4c8e5e961","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.800000000000004,"score_display":"53.8","source_stated_rank":50},"run_fingerprint":"e5950f39fb85fd86d3055449c9ec93953008352b82e7e05f80888651ff0b1fb6","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9f69b25bbb12c75a1ab3bc55b714f90a0b742491ece739fc9735ed1f459c3479","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash","source_effort":"medium","source_model_version":"gemini-3.7-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"chisel","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"5222402bbf4cabc6e8cfc888dc814e1b57a2ee1f632ea18e77a4d139fc717b91","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":43.59,"score_display":"43.6","source_stated_rank":14},"run_fingerprint":"d90604d5d5c1c2b9d43fe78ebe045cabe546ab024a7928cdc6f42c97d1f7ff3b","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"39141ddda9d85ae1eca75b8b14962e1fafe73fe6e753c956146e31a995fb592b","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":3.398069,"date_is_proxy":true,"latency_seconds":1269.021,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.606},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":65536,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"high","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.7-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"3055a769d1c665045f89e8b4be1ce8936ccc71e524153ce8a1e463d3a9b55b82","result":{"confidence_high":74.90075999999999,"confidence_low":60.76524,"sample_count":null,"score":67.833,"score_display":"67.8","source_stated_rank":13},"run_fingerprint":"8d07fa6d993ef3017f0e6e68b1abb534d68dcc02fe086526be489be39afb741e","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c798ba296f0209da3df63f67b225c01d0c8ff628760314b2af3cbf39554b2903","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.040218,"date_is_proxy":true,"latency_seconds":14.533,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.924},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.7-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"26908a60b187dd9473266e77d418c28c23bc160905af272a2a781c39cf92d195","result":{"confidence_high":90.46304,"confidence_low":86.84096,"sample_count":null,"score":88.652,"score_display":"88.7","source_stated_rank":5},"run_fingerprint":"5826b5550e5562cc0120f59fd8feb2078812d5aa683c5b84fc03b3c04b41ec8b","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9e463e29ed717c5cfe164c4f7159867e34531c4f5f25bbb0982ef798981e27d3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (high)","source_effort":null,"source_model_version":"gemini-3.7-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0cc49fd788b243ce3e5b1ffb2e20c37c2ed78e9e36fb1df1508b192966fe1b6d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.828703703703695,"score_display":"56.8","source_stated_rank":25},"run_fingerprint":"714c5415cf8d9f06dccb099cdf76b3480793ff55c0c8c326c7eeeff286fe0e05","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3a3acb8542d1164f65ad058aa7b5689ed7b9fe4c51681e0450dcf0405d68e4aa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (medium)","source_effort":null,"source_model_version":"gemini-3.7-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"de159134f2b5567cc941ce4dca99bcb58a1ff98f6830c9a32d35af6bd91b85bf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.837962962963,"score_display":"59.8","source_stated_rank":8},"run_fingerprint":"2f0d40a2e0ed0de134f7679b1210a21dfcbd2b08c955bd2059c219c8c9775714","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e595e0bb4020231561006955482676611646bdd5e87c1003a8fd9c7f9efea60f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash (low)","source_effort":null,"source_model_version":"gemini-3.7-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"52f69d5fdddaecd30af8d95814131571fa9eb94f33b21c2200910718d7a1359e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.587962962963,"score_display":"53.6","source_stated_rank":59},"run_fingerprint":"de64bcc356327500e001ff6a125642216a705c12b5d592cf3ce0db206a3e2401","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0fbae12e341ffb64b6e97263a4d35f0474cd678b447cd51e0b31301e3d2696f6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (high) · scicode"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash","upstream_model_label":"Gemini 3.7 Flash (high)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"c99233ecc2ebef19b97a18fb88d35c117c95cde0555a929ad1ebafbf36ef6e8b","result":{"confidence_high":62.759438,"confidence_low":51.403496,"sample_count":288,"score":57.175926,"score_display":"57.2","source_stated_rank":null},"run_fingerprint":"b2316dcd9991cbdf5621fccb9a9a4b59477654bf30c36f1aa65097127f772eae","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (high) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"54553656d6654bba0817478f1855484d70889bb5b77cee1b38e1e162de079e07","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · scicode"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-medium","upstream_model_label":"Gemini 3.7 Flash (medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0c8927cd4bacd2561db50536d191a0089d711bbe4857082df429bc94e8c9e2a7","result":{"confidence_high":65.334386,"confidence_low":54.082539,"sample_count":288,"score":59.837963,"score_display":"59.8","source_stated_rank":null},"run_fingerprint":"20f8e05d4a21797aa3faedf16305abbdd054c0d3a3bf0822c2e62362ec645e9f","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4252abe15fac11ba4df1af21a99c1ff5e1783d02d9069243e6a7853c827b9af0","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (low) · scicode"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-low","upstream_model_label":"Gemini 3.7 Flash (low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"aad1dceeefe729b4c9f123b21eb44b7937932333336f5ef8240e12208c54faa4","result":{"confidence_high":61.296683,"confidence_low":49.896603,"sample_count":288,"score":55.671296,"score_display":"55.7","source_stated_rank":null},"run_fingerprint":"138b66189e5cb3ff5bf46909d41caa33ce33fbe4235ef6207bb74832c49f9361","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9badf0f1f2867b7932fb8a561234343b27ab45d006d8afe2cd3d68eff1e49129","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.243628","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3.7-flash-high","source_effort":null,"source_model_version":"gemini-3.7-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"f3311e4be89c1b73ae094ad063587aeffe70b07391548374dd81996c86f2f28c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":904.3,"score_display":"904.30","source_stated_rank":58},"run_fingerprint":"33aeb4552d7ce1c1ba2aa113e953330e19674b943cce434a6c6d2ec16688fc2d","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"00402dc33ae262b1903edad12c857838924ed1632df75d8c3322d67f3e3fdb71","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":4.827643,"date_is_proxy":true,"latency_seconds":1423.494,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.837},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.7-flash"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"d275553b74fde37f548b3669fc5edb65abb9963cf67f2069d52832abbd9414d5","result":{"confidence_high":79.87552,"confidence_low":60.91448,"sample_count":null,"score":70.395,"score_display":"70.4","source_stated_rank":35},"run_fingerprint":"4f2f302fba7b94e30e3a376edf6f485bf71fdfb37fc5afee5b8ae985a4b65992","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"49efe064cf91aa446224aa4c300461e636d3f772c737b1cdb59d2dc3126b34a0","metric_details":{"license":"Proprietary","variance":14.110474300403151},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3.7-flash-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2be9b0344d653caa9fd041de18a563e2a0a97f628be2960cb449aecb87a29e4b","result":{"confidence_high":1599.082275582453,"confidence_low":1584.3574929401914,"sample_count":9624,"score":1591.7198842613222,"score_display":"1592","source_stated_rank":27},"run_fingerprint":"aa6380168d7ae5447cdd9924688a6c2dd6e9c81aef84a5f125ca67d1316c920f","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"101e41e7be035dc21a6fc07a96400d455d9a67b607d26a35a4f95aa5ba936f07","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.7-flash_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"e9ec042a1a861e0f35a880f35fd8924729dcd9dae425ca0c487308b3181f6a81","result":{"confidence_high":1599.36,"confidence_low":1575.17,"sample_count":3007,"score":1587.26,"score_display":"1587.26","source_stated_rank":23},"run_fingerprint":"8e205e30e9b28c9619635fb77fb84d381581eaafb6e803a35a0ee2d84086a356","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d1120e9b89bbdda759c1fa67c935371f90e2cdba64e8261dea6719611e33f18a","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.022219,"date_is_proxy":true,"latency_seconds":492.202,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.649},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.7-flash"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"c8d03fd1ca351744e4480920650a70e0623c55d370e9833fe74cf4c4a4bd8d99","result":{"confidence_high":78.80004000000001,"confidence_low":76.25596,"sample_count":null,"score":77.528,"score_display":"77.5","source_stated_rank":14},"run_fingerprint":"1cf8c8ef85cc310a8750996f150869edeaca32e4c9eeb2a482f89ed7138b4146","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"80dcabc46f13ca33e175aaff7a8bae5f5f284dfac7ab0dd920e09842366db6ca","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (high) · terminalbenchV21"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash","upstream_model_label":"Gemini 3.7 Flash (high)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"6919fbd17a06a087618b493baaaf3c3e0e6d5d464df4d488e7ea31a066ae08a7","result":{"confidence_high":91.547204,"confidence_low":77.028377,"sample_count":89,"score":85.76779,"score_display":"85.8","source_stated_rank":null},"run_fingerprint":"ba8ece32d44ddf5299de6a26f0f189343d3770cb723530d339a56297877d8750","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (high) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"63ff8649be85630b94dd4e72dcd47859d780316ec885b50d65896dbfd1fdb25a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · terminalbenchV21"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-medium","upstream_model_label":"Gemini 3.7 Flash (medium)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"0d5b895fa8845c69d6f940b15150879085ecf18a134a37a869ea82a0c7fb7e1c","result":{"confidence_high":85.576357,"confidence_low":68.637847,"sample_count":89,"score":78.277154,"score_display":"78.3","source_stated_rank":null},"run_fingerprint":"efcd9564f577cfc6529a4aed23f0d490eaee4d16e272980b03279caec0fac46e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9e565cfa9efcb2c803d5b7e93c3f2902c5f8028ac7c4aa29d2e07d79001ba224","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (low) · terminalbenchV21"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-low","upstream_model_label":"Gemini 3.7 Flash (low)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"6e6583c91c8ec54859ae35c6421a34f9172b600eb691ea31054b62a2cebde560","result":{"confidence_high":86.806323,"confidence_low":70.280156,"sample_count":89,"score":79.775281,"score_display":"79.8","source_stated_rank":null},"run_fingerprint":"b32dc5506b14d1d9aff35edad33fa2ef4ac3fce03450fbb5682b371aaf8b56da","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (low) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"17734d69eb642d1dc17b72181b8c7730991d7ef7674bc8fcf815b1774b7174a8","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (high) · tauBanking"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash","upstream_model_label":"Gemini 3.7 Flash (high)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"bdb882a998352194414c2aef9eef02eb80c32a1557da285459625023f8b85807","result":{"confidence_high":42.625062,"confidence_low":24.253686,"sample_count":97,"score":32.783505,"score_display":"32.8","source_stated_rank":null},"run_fingerprint":"2df23e2db8828eba3d20c28679b9c9098932807a4df5b562ff8143a533a5d61f","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (high) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0c77e5ff4465ab44bc93bb5c6e4686ddcafa5fbb9e80bd9ef55fd4a3f1d2a7fb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · tauBanking"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-medium","upstream_model_label":"Gemini 3.7 Flash (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"4033ad290375a877dadbbd617e3c8ee08e7f8176927a85b7d1dbee14170ba0c2","result":{"confidence_high":45.371587,"confidence_low":26.663765,"sample_count":97,"score":35.463918,"score_display":"35.5","source_stated_rank":null},"run_fingerprint":"0bdfd0c6a55ed3cc73d856f1dfb444a9105a3948d76bba738f361ea6bfbaa342","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (medium) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"99022a3daa57407a0c663c1316e971fb2f2754ebf345016dcac1857b478f0ee8","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.7 Flash (low) · tauBanking"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-7-flash-low","upstream_model_label":"Gemini 3.7 Flash (low)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"bd992dfbe7177c07d585b37299574e91467d731bb0699eac2c13209702f1d2c7","result":{"confidence_high":39.200035,"confidence_low":21.332127,"sample_count":97,"score":29.484536,"score_display":"29.5","source_stated_rank":null},"run_fingerprint":"587b677cd3adec4a9bc0c240e01e6f7da5b6971d9061e36d9178eb3de4532490","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (low) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"824e7818f163a9345cebf943a3d81e56379e7bf66c55ec8617bd4741e2684973","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.794","mean_standard_error":"3.8","model_release_date":"2026-08-13","notes":null,"standard_error_points":509.99999999999994,"upstream_source_url":null},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.7 Flash","source_effort":null,"source_model_version":"gemini-3.7-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"17837c7019baf3dc13f5839c1ae93490304b8568467dadb72647eb637af82247","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":67.8,"score_display":"67.8","source_stated_rank":4},"run_fingerprint":"1b330e4d5bac85d5edb9e18b15fdccb4ad85168d18ee941de65ab2c6bc7de63e","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e5060d9b2625a987949324dd","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"30801690f17b23b7f6503e39e4a86e751dc4a6af4b6dc4e680f72e9af936ddb0","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":54594},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Gemini 3.7 Flash (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ec585067133de1255fc972d31322b198e63bcd033cb8fe7c9aabb961c8b0867d","result":{"confidence_high":-0.007497656197917133,"confidence_low":-0.022204972726663337,"sample_count":5048920,"score":-0.014851314462290235,"score_display":"-0.0149","source_stated_rank":36},"run_fingerprint":"ea5165f6776316dd148261d37e53fe12fdf70e9ed725df097c65d36fd2d7c41b","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ddd7f334fbb61cb16f37493f3eef6140f77dc291162e1e6bebe7e4e31b0be4a3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Gemini 3.7 Flash (High) · gdpval"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-7-flash","upstream_model_label":"Gemini 3.7 Flash (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"01ce08d953a0fcdae5232d3659aa2b61e8d00fffbdc0f0c6a895afe4c6f02632","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1371.25,"score_display":"1371","source_stated_rank":null},"run_fingerprint":"037b680da328da5682a2a9b33ef961bc719c4e92fed42dc2ffb2eb328b110649","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_45fbb04067e2dc09bef71f8c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5e865c0151c97b79f9df834f99ff0c8fae761e27b185b95721a31efa8a7e9225","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","reported_confidence_interval":"-27 / +27","source_locator":"Leaderboard models table · Gemini 3.7 Flash (Medium) · gdpval"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-7-flash-medium","upstream_model_label":"Gemini 3.7 Flash (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"67bb74749d7791336252ecd7017bfcbf0db8720e39da386b31904c49259a7033","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1339.58,"score_display":"1340","source_stated_rank":null},"run_fingerprint":"d10071799af094f663262801416fc077a39f88769fd8288039de6bd08cbc50c4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ad3d84b960dfc393e7214ac4","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"845e8727d86af86b7763a32ea368b881bd0780a73a86517adb2ffcc46d139432","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-13","reported_confidence_interval":"-27 / +27","source_locator":"Leaderboard models table · Gemini 3.7 Flash (Low) · gdpval"},"model":{"id":"google/gemini-3-7-flash","name":"Gemini 3.7 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-7-flash-low","upstream_model_label":"Gemini 3.7 Flash (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"ff017eb350f518e62dc1e48cd33fbf3e81e54bed3924eb88c62669c20e3ff75e","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1305.23,"score_display":"1305","source_stated_rank":null},"run_fingerprint":"f6ca9cc2fa58b9cc8be91e025396fd55c9b82d4898a21d017237eb19b2817d29","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.7 Flash (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_1497641e9f2d797cc3038c99","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"afe82cf98779207f08080b8e593fc0f9ef5d292d5e7e222b43091ec2241ba2ae","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-09-02"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 3.8 Flash; model release: 2026-09-02","source_accessibility":"API access","source_model_name":"Gemini 3.8 Flash","source_model_release_date":"2026-09-02","source_model_versions":["gemini-3.8-flash_high","gemini-3.8-flash_medium","gemini-3.8-flash_unknown"],"source_reasoning_efforts":["medium","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ae90a7f6912a59848644b4b4445b1edeb5605feb33f4a764378a51540d7fc0c1","result":{"confidence_high":160.42,"confidence_low":154.64,"sample_count":null,"score":156.93,"score_display":"156.93","source_stated_rank":15},"run_fingerprint":"5ff5be6a4d0757e948c0586aba6d27548593924723d3790dc8974dc14503e741","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e5abce6ad7b99a4c5dd13f88","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"606b1f560e3d3d7096c39c398828b33dec30ba075272fe6f7937648c3e556f0b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (High) · intelligenceIndex"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash","upstream_model_label":"Gemini 3.8 Flash (High)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"b752bfa0a14c9e4e042c61475a63d1261a6c493ed6b8b7ee2a6c1bd591254c73","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.926232,"score_display":"40.9","source_stated_rank":null},"run_fingerprint":"8d9b12779209c230faf012f95cdd7bb2c37506a9a392122b033df5ea9ee89dc2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (High) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"90ba1dfbde9f40aae91e9fee17c21d12e0a255cfd27a5396fac4d745cc2b7279","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · intelligenceIndex"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-medium","upstream_model_label":"Gemini 3.8 Flash (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"295b6e504612eba3f9fd8805080f4353d8cb41d03c1dbd1928d247525559eddd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.774019,"score_display":"39.8","source_stated_rank":null},"run_fingerprint":"0cfcb50e9df4a9cb82f28be61d2405002bfb5ea1754e22bdbe54f3ec70c0fc67","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a07a3c60973cab1d1c4d3a22c0839a2bfbd317064e14ce6530bcbe5739956a0d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · intelligenceIndex"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-low","upstream_model_label":"Gemini 3.8 Flash (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"00edf7a91c3930e54a766b456dca9d9f71d74dd3dadc02d9cabec07e6ed7ef3d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.454811,"score_display":"33.5","source_stated_rank":null},"run_fingerprint":"42e324845d600a67fa6c9c47c67ef4d7024cade0bc22907d0e1c235b784eded8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f2ec3a59dbb5c6fbdf1ea2b08cd239ae9798bd1adbb456e4cfd7f96ece7cb368","metric_details":{"license":"Proprietary","variance":6.620856874344741},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1499.8167579780886,"confidence_low":1489.7303812172283,"sample_count":26298,"score":1494.7735695976585,"score_display":"1495","source_stated_rank":8},"run_fingerprint":"8af54b42fc773ed20dc5d7e4300d174da2c59ef94c0d4f8bb48f1b15bd4ca8aa","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T19:01:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"9d0346088f086c8ece880eaac73baf8c0781038a93513df67688a5082a2eb608","metric_details":{"category_scores":{"Agentic Coding":54.242333333333335,"Coding":72.489,"Data Analysis":54.00566666666666,"IF":81.4125,"Language":87.79266666666666,"Mathematics":91.56425,"Reasoning":89.29325}},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.82852380952382,"score_display":"75.83","source_stated_rank":34},"run_fingerprint":"e8b8619096b0b479dac9692a6bc6b1d11a022b24a2a9c1efaebd06cd60be1f8d","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":25,"at":"2026-09-09T19:01:48Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-09T19:01:48Z","status":"fresh"},"measurement_id":"a6e0e1f1f7f8268fe019496dea6986710c57f79c606518822b8a04b58f13d304","metric_details":{"calibration_error":51.0,"confidence_interval_half_width":1.96,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=Gemini 3.8 Flash; createdAt=2026-09-09T19:01:48Z","source_row_created_at":"2026-09-09T19:01:48Z"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"Gemini 3.8 Flash","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"716dba445a290d26f118ca23c5b039f25b88b4d42d18a2da84735b24d21327c1","result":{"confidence_high":46.480000000000004,"confidence_low":42.56,"sample_count":2500,"score":44.52,"score_display":"44.52","source_stated_rank":2},"run_fingerprint":"bc72b9105639acbb8fc7bfe51ba426b3ff165621aa60409a175c930096f92b70","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=Gemini 3.8 Flash; createdAt=2026-09-09T19:01:48Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_491d6d7d0d5b0ccd98ec2ae4","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f50d61d64bb087ee3cc4fe86ad98599a85766db18747a89cf4183cdfb8a13a4d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (High) · hle"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash","upstream_model_label":"Gemini 3.8 Flash (High)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9f55f5639e4e267534bee456e9668a7b2840909cd2fbfcafdd60d613dd242c39","result":{"confidence_high":49.931656,"confidence_low":45.720198,"sample_count":2158,"score":47.822057,"score_display":"47.8","source_stated_rank":null},"run_fingerprint":"af87f97b904adefb63710d89f2525e92b7eb5f20c7bdc1bb69c6bddaec7288e5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (High) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e4154803689a3e83221180666afbd89f9ce8729d6222af9b8648c4f87a393e1c","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · hle"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-medium","upstream_model_label":"Gemini 3.8 Flash (Medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"381fa1d8f79a27f9d7bc95e0048b98323747747d2b0e9e2a79a4d9c4fc30bc9f","result":{"confidence_high":44.217782,"confidence_low":40.054885,"sample_count":2158,"score":42.122335,"score_display":"42.1","source_stated_rank":null},"run_fingerprint":"7323d7d04157c9ca4b5cad502f539566cfe1db503a75601784ba794018acda34","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4b02afa82e01ace5cb7fb13da2906336a772d3010c30b487cc6d783dba42f21c","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · hle"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-low","upstream_model_label":"Gemini 3.8 Flash (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"12961f3cc0084440b72e7b704d111aadbaf33de448e26e01044eba0749267f8a","result":{"confidence_high":39.130512,"confidence_low":35.058161,"sample_count":2158,"score":37.071362,"score_display":"37.1","source_stated_rank":null},"run_fingerprint":"02ae2f1a16f26878e5598e1d392c7c3d597c413542e84373b65fadcdf1bb4b03","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T17:15:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T17:15:36.000Z","first_observed_at":"2026-09-09T22:25:13Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f072abfecb8ced03a7df2c0cd36743870b19d525ff6a135194818b2fc62d5005","metric_details":{"best_score_across_scorers":"0.697","model_release_date":"2026-09-02","stderr":1.4539683710535192},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gZ4bwozTKtpmZTqnqX3a33","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"addbe5e3a24d08a8962ffa2f57bb6b6881faef74d4cde0934b7d57db69e9d41a","result":{"confidence_high":72.54977800726489,"confidence_low":66.85022199273509,"sample_count":1000,"score":69.69999999999999,"score_display":"69.7","source_stated_rank":7},"run_fingerprint":"6c283f0164dd9e31a382bc7c7fe36ff555432d429fd2deee98aa74252e76d345","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":28,"at":"2026-09-06T22:24:07.699781Z","basis":"evaluation_at","evaluated_at":"2026-09-06T22:24:07.699781Z","first_observed_at":"2026-09-10T20:32:03Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4a00acfa3a939cc84f522af55bf441d1a63456188ff34fc915926b6d283ec666","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.8-flash","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e87264b0d318e703f7ec29aeb2ce6888dfe87424ea075fb0f12822e8a68f34ca","result":{"confidence_high":77.32995098197088,"confidence_low":71.84298134885617,"sample_count":null,"score":74.58646616541353,"score_display":"74.6","source_stated_rank":3},"run_fingerprint":"e6fcea116fe5ee4f4baba63475c15f715bec42833f17065307b710256d1e695d","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e363d8d801ebce4fc74ac9eb","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"90482f6157dfe957998044e0b769e899dff35a41535b1abf9ce58332f2c1c5bf","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.056302,"date_is_proxy":true,"latency_seconds":20.18,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.477},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":94.949,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.8-flash","zero_shot_accuracy":93.939},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"48f6f39bac9672004928e77641d71d8766ad44908528eeb8acccb094b953e62d","result":{"confidence_high":97.33892,"confidence_low":91.54908,"sample_count":396,"score":94.444,"score_display":"94.44","source_stated_rank":4},"run_fingerprint":"bd84468bf5c24177c3621645382e9d7efb13493fa4ea3930507b91f52c36f380","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a5341caa633c7abbe221701766ec41d95d210ffc664db8c961ddc1e140de5578","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (High) · gpqa"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash","upstream_model_label":"Gemini 3.8 Flash (High)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8daf1862f736c5caaa9fbde0b1ee6043afe7e639591cb2d22ad23d62f1d5a714","result":{"confidence_high":97.44879,"confidence_low":91.3337,"sample_count":198,"score":95.252525,"score_display":"95.3","source_stated_rank":null},"run_fingerprint":"108fed0f57287b8d934e9072e04d690e7b56e7420b0e0e09db915ab6679e8389","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (High) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0fb2e55c7d9a31fa98b5fc83e26cc2d1402bfc28f6a1b5d6319a42e7bff03d6e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · gpqa"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-medium","upstream_model_label":"Gemini 3.8 Flash (Medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"85cee2addd796b6e5433fded6b8be3f2b9c6acdffcd5284196923ca22b054492","result":{"confidence_high":96.198914,"confidence_low":89.2146,"sample_count":198,"score":93.535354,"score_display":"93.5","source_stated_rank":null},"run_fingerprint":"6c956fcd57140564bf3ec5fe11af303e2722cd32ecf11d661b1c8961f8597e55","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a75e8c0a1f75bd35e760c6e608a63b27e3a1d3c0084377b3aecee12b638d58f3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · gpqa"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-low","upstream_model_label":"Gemini 3.8 Flash (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"28bfe547a86d7056318091c2b035b391b3b7d9915d2f31e2fad3f6a4530fb720","result":{"confidence_high":95.043453,"confidence_low":87.397431,"sample_count":198,"score":92.020202,"score_display":"92.0","source_stated_rank":null},"run_fingerprint":"bd85223e306c938647226cc909d7b41c74fe4fbad1fbef7adaf5bd60b7f35c16","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":32,"at":"2026-09-02T17:15:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T17:15:36.000Z","first_observed_at":"2026-09-09T18:29:51Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4381554cd60abcdf42292be58d76a86d5611d49b9b8653d5213387de254b3170","metric_details":{"best_score_across_scorers":"0.9539141414141414","model_release_date":"2026-09-02","stderr":1.401136318210652},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Br3mjs2GrYcCwX9AMwcihU","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"36e018a26a52a7574eba23dd0a028b29bdc9e0bdd63169bab6e98558c982565d","result":{"confidence_high":98.13764132510703,"confidence_low":92.64518695772126,"sample_count":null,"score":95.39141414141415,"score_display":"95.4","source_stated_rank":3},"run_fingerprint":"43ccd07c4afe28da6822c60de41e2b784f9d54b65f143ba18e031ce01492a674","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-03T18:01:00.901075Z","basis":"evaluation_at","evaluated_at":"2026-09-03T18:01:00.901075Z","first_observed_at":"2026-09-10T20:32:03Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"58846040233b5a43c1f08f0d86841069269bec680c8cbf8102d3d95eaf2a37e3","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.8-flash","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a349d3db260ac3f1b29d8cc11b266a06dd85a321d86150c6251de21f41057a67","result":{"confidence_high":95.31638210958802,"confidence_low":87.5119007186948,"sample_count":null,"score":91.41414141414141,"score_display":"91.4","source_stated_rank":6},"run_fingerprint":"4369117174685060401ff9922ce30ab2d5f69d670454f4b2aeeff0f8c1c57b8e","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4c8050fb5cd4f1ccb31e659c","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"718cb96edadaa702de556ead2a4dda62e0c1c9400cca0061bdbc117b70b5c189","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.022457,"date_is_proxy":true,"latency_seconds":7.82,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.295},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65536,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.8-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"190085f02ed9b5f9f219b69f23f71511ed53adc2afc9e2bd8785ceec764a3f8d","result":{"confidence_high":90.7992,"confidence_low":89.64280000000001,"sample_count":null,"score":90.221,"score_display":"90.2","source_stated_rank":5},"run_fingerprint":"2848d69e64a95101ce7b4dfd02bb184c374c252754dc87f5977f6814c1b279e3","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T17:15:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T17:15:36.000Z","first_observed_at":"2026-09-09T18:29:45Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"94c7c355ee0185d461f8c20ead0984e9fc5c768cb4c327154a682d9ef70f597d","metric_details":{"best_score_across_scorers":"0.9888888888888889","model_release_date":"2026-09-02","stderr":0.8724007409153889},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"NeYuqgQyWtFWAHEjQYPm5Z","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b42224bdd5801441ea632d9fc7232b935c6196270e058988ee5a4bffa469c6e4","result":{"confidence_high":100.0,"confidence_low":97.17898343669472,"sample_count":45,"score":98.88888888888889,"score_display":"98.9","source_stated_rank":17},"run_fingerprint":"b0701740dec40671708871688b9c5959ebf055a73b5e6aedc2f4d2ba878017f5","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T17:15:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T17:15:36.000Z","first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"56e32e10dcd72899b808239307716c6444417796d6e8cc236e25c2303a15e67d","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.6842105263157895","leaderboard_private_problem_count":285,"model_release_date":"2026-09-02","public_example_count":10,"stderr":2.7582557383453006},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"Utgs6DuaQDDdi98ZWH22qL","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.8 Flash (high)","thinking":null,"upstream_model_id":"gemini-3.8-flash_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"fd8b819ddb1e17afd50e047bcc1b8c90c78e246dca57235b7fcbef49bd394df9","result":{"confidence_high":73.82723387873574,"confidence_low":63.01487138442216,"sample_count":285,"score":68.42105263157895,"score_display":"68.4","source_stated_rank":26},"run_fingerprint":"fd3910a2e5a7af86c99c1331d34766b25d90d6ed1160c8f7067c46a10bea41ba","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bb09b1be77148c1de9b69a3dc0fdc61eea5ef83ff4462c5080957e803723cd2c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.39972941250000005,"model_release_date":"2026-09-02T00:00:00.000Z","results_url":"https://arcprize.org/results/google-gemini-3-8-flash"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"google-gemini-3-8-flash","model_type":"CoT","upstream_model_id":"google-gemini-3-8-flash-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4adc9966b4f2a764c90a6ac1a6cf7f028c814f9d7576eabe5c7198b7d98f41c8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":89.16666666666667,"score_display":"89","source_stated_rank":18},"run_fingerprint":"bb93238aa57b6077f70a462e92eec7d627d98589e4e0d169db35d57427fe3b1d","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b8e4ed91d13b78ceb2531d6aed058c0af76623659b95eedcd86209b9b4f503eb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.2477876749999999,"model_release_date":"2026-09-02T00:00:00.000Z","results_url":"https://arcprize.org/results/google-gemini-3-8-flash"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"google-gemini-3-8-flash","model_type":"CoT","upstream_model_id":"google-gemini-3-8-flash-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1f4caafa749d68941b321ae5fe40db87b79e286efb06feb4eb8326f626347728","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.91666666666667,"score_display":"83","source_stated_rank":37},"run_fingerprint":"c1db2df392286d03b31fe00f194a3fe72edac441651638176d5fe61edcbdc942","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c294045da862cf9b351dc185310583898cd5b0b71102db5c65f359302a6cbb6e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.15445401874999998,"model_release_date":"2026-09-02T00:00:00.000Z","results_url":"https://arcprize.org/results/google-gemini-3-8-flash"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"google-gemini-3-8-flash","model_type":"CoT","upstream_model_id":"google-gemini-3-8-flash-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"abfa5396135c716240a774be9021ed9705ce9ebd76e7c66dd213929a35f29941","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.5,"score_display":"78","source_stated_rank":41},"run_fingerprint":"eaba3cde0fc2f487d47f28a7b60a0dc05693f004580c13e0c13f515dfd9746bb","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T17:15:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T17:15:36.000Z","first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6104a9253d444ba193c0e7072d6cf64caff051b97cdb530d842a886b9298ca25","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.21951219512195122","leaderboard_private_problem_count":41,"model_release_date":"2026-09-02","public_example_count":2,"stderr":6.5445892024384085},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"Lgy52ZT4YpFrUqqKVu2qUs","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.8 Flash (high)","thinking":null,"upstream_model_id":"gemini-3.8-flash_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"bb63df4a7cc1a4736fced56d574fdc35e3518f8449aca663fff044b6db3d874a","result":{"confidence_high":34.7786143489744,"confidence_low":9.123824675415843,"sample_count":41,"score":21.951219512195124,"score_display":"22.0","source_stated_rank":44},"run_fingerprint":"9a775c783c4fcf6d519668978155f380c51c77e3e4423a450230b7dfcc9ef9fd","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f2ab65111860fc5e5602c714f49ed6be22b0deaf2e900943afedaca2266ad782","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.004472,"date_is_proxy":true,"latency_seconds":12.028,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.75},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65536,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.8-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"297298b1bc3167d449947fa29371a6f4e5ebb95c09607e248d70d47e79eefcc8","result":{"confidence_high":90.545,"confidence_low":87.605,"sample_count":null,"score":89.075,"score_display":"89.1","source_stated_rank":4},"run_fingerprint":"ff7f47d6e196ed6362b56b25715f2ba2eddf4352acac013d321ecb257349eb8f","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cc8f7b8dc38248782dbbd0b37531cb9e53cb125567ed09768683dc9e71c30f1d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (High) · mmmuPro"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash","upstream_model_label":"Gemini 3.8 Flash (High)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a6468f062f17b6b3376c3b4d593f5f71a53ef48b83e93aba0074a7eed025c543","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.606936,"score_display":"85.6","source_stated_rank":null},"run_fingerprint":"578cbefd4ef55b21ef8363da4a0aa17ba96247c6f2f4b5fb05bbb4d864467b6f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (High) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fbbaa76a31ea4b08430e952e40e70a3476ff81a117a640e70bb85fa0841c1afc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · mmmuPro"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-medium","upstream_model_label":"Gemini 3.8 Flash (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b0a2cb0dcc8804ecdb2264829449d518293154a78ddd01c183f7a3cd3d70e980","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.219653,"score_display":"84.2","source_stated_rank":null},"run_fingerprint":"448eab2d31c804ae050c3f2a800b56f74bb55b7b65ad7eb3251bf5e970ed3024","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7681ccf67a1567b78786299e66f514a58123033e64c8e494c3c41165e4399ed0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · mmmuPro"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-low","upstream_model_label":"Gemini 3.8 Flash (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b9dbab736adfa34528ceb007a8aef733b406b2ff517343be17947e6a62be8202","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.508671,"score_display":"84.5","source_stated_rank":null},"run_fingerprint":"c171c61bafbc0ddd35f18a24e5f025b23c7e187c1ab7087de088c9d7d70b4268","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0a49c30b4b6c26dafbb3c1e738b7fd0b12fe9c64b8ad3801708085d06159ad81","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash (high)","source_effort":null,"source_model_version":"gemini-3.8-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"40bf9ac21c272be5e5f8bf328c78d71841156733b2ed26ac96810d1323ad130f","result":{"confidence_high":28.821301,"confidence_low":11.0059,"sample_count":71,"score":18.2857142857143,"score_display":"18.3","source_stated_rank":59},"run_fingerprint":"183ad0f6c64465a6e26d35db77250f0e935e3c0c93ae16860ecf387c0061dc44","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cac9cb4baf3188cdb0f6ad4d9105ce56771aabbd39a6dbba716eb64b6c0c12f1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash (medium)","source_effort":null,"source_model_version":"gemini-3.8-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7ca36552dbdef9d95a79450ce73a7768766c545fe560c651b79391c713caeae6","result":{"confidence_high":21.906771,"confidence_low":6.536387,"sample_count":71,"score":12.285714285714299,"score_display":"12.3","source_stated_rank":86},"run_fingerprint":"f3b46b6bc75e93402819e0be3c287ac1dbffd6ef84f1fc946a40164ff0d280fb","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bef8a55de5560d103009d0c5286faa523370e56f11321d1b86d145bb6f51f7e9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash (low)","source_effort":null,"source_model_version":"gemini-3.8-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"25486782bd8b60e7e856b837c824590ddc5e1f47378712d88ba3cd9260619427","result":{"confidence_high":11.389661,"confidence_low":1.332676,"sample_count":71,"score":4.0,"score_display":"4.0","source_stated_rank":114},"run_fingerprint":"5b1bd0424bc41734a89a8e7d2c25bf0e91ba329137b162808fc74a45177ee449","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"958f51ed82c01293175399d9a4e8de6a103752096541491de033c067951c7084","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (High) · critpt"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash","upstream_model_label":"Gemini 3.8 Flash (High)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"08b5d7a9a48576b6da71a6a083cbc7be70fbd3a767cfd590d244f602a800d9c7","result":{"confidence_high":28.905487,"confidence_low":10.965804,"sample_count":70,"score":18.285714,"score_display":"18.3","source_stated_rank":null},"run_fingerprint":"cabd666ffc4f23fb424d6bd8ce1df48114ff1dd0a8cbe79a154dfac60d55f164","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (High) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"787830c9fff1d49f353b745afffa765615b7dc53513cabcb9109180610ecbd6b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · critpt"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-medium","upstream_model_label":"Gemini 3.8 Flash (Medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"9aff7347ff6ac0be6cd24459d1c3ccc02933114e9d9f9199ec020ad6bfa11711","result":{"confidence_high":21.988174,"confidence_low":6.507416,"sample_count":70,"score":12.285714,"score_display":"12.3","source_stated_rank":null},"run_fingerprint":"57eeb53fac7a2c4fef91b84eefc5be832d313b1e1e2a710e40423229f1266aa1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1b027315af039e0a1727a99645e52cc945cbfd0801101499a4b7706263fe2dc8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · critpt"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-low","upstream_model_label":"Gemini 3.8 Flash (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"f7674c73d95da863bf9ea0d889633c6538775def66e30e029649a73b481fad02","result":{"confidence_high":11.463124,"confidence_low":1.323165,"sample_count":70,"score":4.0,"score_display":"4.0","source_stated_rank":null},"run_fingerprint":"24483cac75f1515f6f167401bc2550204e9a1f6ee4ac600c669033419fc68fd1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T17:15:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T17:15:36.000Z","first_observed_at":"2026-09-09T18:29:48Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3de69f3c5c4dc16f2627cdb7a89802e29a3c347111a6c222994f05c0bdde9fdb","metric_details":{"best_score_across_scorers":"0.61","model_release_date":"2026-09-02","stderr":4.902071300001973},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BCCBjMXW674jF57sP4jf2h","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"906f38bbbc8bf82b82a6bcc8b6052949ffc48c9b238858406a4ea425a858e67d","result":{"confidence_high":70.60805974800387,"confidence_low":51.39194025199613,"sample_count":100,"score":61.0,"score_display":"61.0","source_stated_rank":4},"run_fingerprint":"bea9862c52ecac50827da96f0d6a59544f8ae9af7da27b2ea20ba30ae17bb7a1","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"896b953e1338dda7ddd38d3b1501f848dd1ef2b94dd52e2a04f19a938073ac84","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.1911,"date_is_proxy":true,"latency_seconds":683.446,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.791},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.8-flash"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"d09e0e354f75fd8df287fc835922276328390a4cc1d30ef78b9ce22985146094","result":{"confidence_high":83.51036,"confidence_low":76.48964,"sample_count":500,"score":80.0,"score_display":"80.0","source_stated_rank":24},"run_fingerprint":"d1f03acb3e28d828ecd82e8b4c8a2bb311820d82cb7340d01438713b39a65947","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T19:00:09Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T19:00:09Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"0ef64a9deb630477bdb9934a18e2c80865dcbb13eff92446b3e52ae56c9edf2d","metric_details":{"ci_attempted":447,"ci_half_points":1.4173132066358782,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":330,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":166.3131991051454,"mean_cache_tokens":21456039.230425056,"mean_cost_usd":2.362349413758389,"mean_duration_seconds":686.7691519843399,"mean_input_tokens":21734449.25279642,"mean_output_tokens":143242.6644295302,"median_agent_steps":161.0,"median_cost_usd":2.1098912999999997,"median_duration_seconds":604.643838,"median_input_tokens":18026661.0,"median_output_tokens":138377.0,"median_output_tokens_to_pass":136617.5,"median_peak_context_tokens":215558.0,"n_attempted":447,"n_passed":330,"n_tasks_attempted":113,"n_tasks_passed_any":97,"pass_at_4_points":85.84070796460178,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gemini_3_8_flash_high","source_model_version":"gemini-3-8-flash","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"bb706fd403a809640e72b3480dd7b01fc30d05d6a6ef39a0eca2982aad7ea2f5","result":{"confidence_high":75.24281656234058,"confidence_low":72.40819014906883,"sample_count":447,"score":73.8255033557047,"score_display":"73.8","source_stated_rank":2},"run_fingerprint":"f991d213bad7176a6025690307b002e94cf1fc3203f20c047473105c96d32976","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T19:00:09Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T19:00:09Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"b3aa9037afb8e37ddf11c2c58f285f2f14274aa6d71277a05f4bf48a0fa4ae0a","metric_details":{"ci_attempted":452,"ci_half_points":2.280804572877925,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":321,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":147.30088495575222,"mean_cache_tokens":16519023.008849557,"mean_cost_usd":1.9671024789823006,"mean_duration_seconds":826.5496385464602,"mean_input_tokens":16863939.814159293,"mean_output_tokens":124684.36283185841,"median_agent_steps":140.0,"median_cost_usd":1.7342966249999998,"median_duration_seconds":668.1031395,"median_input_tokens":13458718.0,"median_output_tokens":120488.0,"median_output_tokens_to_pass":119834.0,"median_peak_context_tokens":189491.5,"n_attempted":452,"n_passed":321,"n_tasks_attempted":113,"n_tasks_passed_any":94,"pass_at_4_points":83.1858407079646,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gemini_3_8_flash_medium","source_model_version":"gemini-3-8-flash","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"d8c598269b5d393079566e3974ac04bbeed48cccc2d075ee011886edf7473f7e","result":{"confidence_high":73.29850368792218,"confidence_low":68.73689454216633,"sample_count":452,"score":71.01769911504425,"score_display":"71.0","source_stated_rank":10},"run_fingerprint":"325641b4cf77b42717a7546246f908b01cda2dc0d94d69273450731aa7c547bb","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":32,"at":"2026-09-02T21:26:37Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:37Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"12ab571956d34b6d8549d40afef9e85ed2d5a958f334351f35ef889d4ac3fa87","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"166.3131991051454","mean_cost_usd":"2.362349413758389","mean_output_tokens":"143242.6644295302","model_release_date":"2026-09-02","notes":null,"pass_4":"0.8584070796460178","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-8-flash (high)","source_effort":"high","source_model_version":"gemini-3.8-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"3539f513fdf6a88f160a08111e7dd9a5711c7f398a01b5646f8ea2a7c383197d","result":{"confidence_high":75.24281656234058,"confidence_low":72.40819014906882,"sample_count":113,"score":73.8255033557047,"score_display":"73.8","source_stated_rank":2},"run_fingerprint":"640d94c9fc2bfa37a4c22f7ce3da2c6f3363c30574e843b35d74b8ac6434ab6b","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":32,"at":"2026-09-02T21:26:37Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:37Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dbe593d683bdf26e533eb1e82ed39683d136999bc4ea1b40267b65e86e16ce51","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"147.30088495575222","mean_cost_usd":"1.9671024789823006","mean_output_tokens":"124684.36283185841","model_release_date":"2026-09-02","notes":null,"pass_4":"0.831858407079646","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-8-flash (medium)","source_effort":"medium","source_model_version":"gemini-3.8-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"bd6e568d30d674d32b271d20b2a6cf23476747c2d182c91727b714f3d8a7a87e","result":{"confidence_high":73.29850368792218,"confidence_low":68.73689454216633,"sample_count":113,"score":71.01769911504425,"score_display":"71.0","source_stated_rank":10},"run_fingerprint":"bdfd7bc19a8125b27a0876359c29dc4ca8ead3912908c9f79c5391e7db46e47a","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"779ddbf0118ae8db6439135a2671555ad48692f0c0e63b62e2fd63079488de79","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.7","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"steps_per_task":"324","tokens_per_task":"162565","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash High","source_effort":"High","source_model_version":"gemini-3.8-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"bcf33621637fd790f9e0b54377fa086de7ddba2726b1c04914adf0fbde41cfe4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.6,"score_display":"39.6","source_stated_rank":27},"run_fingerprint":"3b5e7a6c51cada057c5a87deeb5dc298a396e09b9334460796701884fb11453c","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"491df5975a0a622d0a77ee714f971a97037f0a34cc2669139208ca6a7e0260ef","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.06","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"steps_per_task":"290","tokens_per_task":"128364","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash Medium","source_effort":"Medium","source_model_version":"gemini-3.8-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"f7e46544014c333121d0f6690f11bc52718714ed555de3cde27b41314327e261","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.3,"score_display":"37.3","source_stated_rank":31},"run_fingerprint":"b75913284e41a7952e6869925fc9b20a424045c310a7acd3adb730ece0201827","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d4b644b7c478c8741ac9053b21b865335e695e6a020dfcd90d59856f774bc170","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash","source_effort":"medium","source_model_version":"gemini-3.8-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"chisel","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"dcbe1b014336dec05353a454e12ffb1b8f0f23c9adda0c1a7a2d16c94076347e","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":41.19,"score_display":"41.2","source_stated_rank":21},"run_fingerprint":"4c5774d061bafc78f58b009645ea927196afe3eef5ee164c42f2c8d03d6d6960","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c9534649d7507a06dc9009e5cdef31c12a38caa6462d8ad08f633771581a519d","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":3.976052,"date_is_proxy":true,"latency_seconds":860.465,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.257},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":65536,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"high","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.8-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"c822ad8d759c96bb32ce6484424aef690ebe1b1eddc07bcc3f94ca5f3788fa4f","result":{"confidence_high":61.367720000000006,"confidence_low":52.52028,"sample_count":null,"score":56.944,"score_display":"56.9","source_stated_rank":17},"run_fingerprint":"375516336f79a5fbe1a7061dc4321d7981313ba1ca9199b737bbb1dd1a73fd21","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"abd75ea6915ab45f0c4a522a243943511a1e79225b5e97474726ecfbf1feb47a","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.07124,"date_is_proxy":true,"latency_seconds":24.07,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.897},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.8-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"1d0db59b8e47af6bb9928f6509d449ca2e5664ac23293534f8404e901a6d7d21","result":{"confidence_high":91.24212,"confidence_low":87.72587999999999,"sample_count":null,"score":89.484,"score_display":"89.5","source_stated_rank":3},"run_fingerprint":"e1440bb8e747fd514ed42e1e187af94cfe0c5393bf7002d74dea76526a17eb70","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9c12a3932afb15fdd1f9342c31c730907036f703afeb5c053fdffd6d1feafa6f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash (high)","source_effort":null,"source_model_version":"gemini-3.8-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"866574c27d207bb0674e3ecb923da8cea0d9ca785e79e75a248904a21012f0b9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.5972222222222,"score_display":"56.6","source_stated_rank":26},"run_fingerprint":"e21271535304cd8f0eb9a1149909fec8dcc956d06c1ac568bd18ad193416638c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5134e4802462796f5ad89ea3a54bdab47871b2398cf790be45a4bbe0df790657","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash (medium)","source_effort":null,"source_model_version":"gemini-3.8-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b44a82af5fa774109fb56839e9d86a38dedb3a5e0867700b699c0d8c1715474c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.398148148148195,"score_display":"54.4","source_stated_rank":50},"run_fingerprint":"48e27de0a160146dd6810e011e0b3d68fa84861f23006e532bbae629212d4036","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"97473198dfff7159e6b8333b44d6b03fef0d679f183e22ddfdcb473af6097a0c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash (low)","source_effort":null,"source_model_version":"gemini-3.8-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2c3f6249df65b2750da6e144ff3a2f46bbeb04353f3d0b6186beb122bc935468","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.2824074074074,"score_display":"54.3","source_stated_rank":51},"run_fingerprint":"cbf519d0869d97c03e4196c26b52f98682a95ace36416c72f3aba8afc9bc8878","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d6709a795cd0b75b90d9ae22971bb13af7e4a4d95569f5827d4a7c84f4a3212e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (High) · scicode"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash","upstream_model_label":"Gemini 3.8 Flash (High)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2fd362e1261fcad8bfb55548e5c2257008aa16f1f221d752a513bf491e759dee","result":{"confidence_high":62.197462,"confidence_low":50.8233,"sample_count":288,"score":56.597222,"score_display":"56.6","source_stated_rank":null},"run_fingerprint":"c1434cf1ec5206370cffb8c2d9a57a25b1438525c3fe8545494275258e7d48ca","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (High) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"270fe3a965292026fd6498128072797ffc51434dd0510ac47755ca8e193fa77e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · scicode"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-medium","upstream_model_label":"Gemini 3.8 Flash (Medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"8a1ee174c3f3c47270a9032bf7c0c89010eff1ab888b7907dd6840e3871be62a","result":{"confidence_high":60.732691,"confidence_low":49.318424,"sample_count":288,"score":55.092593,"score_display":"55.1","source_stated_rank":null},"run_fingerprint":"6a6c255694c853c5498567a82cfb75eb50cca2846472f20445b0c902ed33aa69","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"63984c5d6acb723606f501c640abfbce8b2402a64650037d7dce0a619d141872","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · scicode"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-low","upstream_model_label":"Gemini 3.8 Flash (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"231ab090c99bbbce42e45e1a9ac90774a10b740cfe0fa54e67eb32d1469694b5","result":{"confidence_high":60.6198,"confidence_low":49.202881,"sample_count":288,"score":54.976852,"score_display":"55.0","source_stated_rank":null},"run_fingerprint":"d1917463526d9e283a397ba7a719d095770775e168780fe21894ed140205086c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bf20386cac1377f350616188bc34a0c01653994c9c82b0d6039eceddbb2c3880","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.344275","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3.8-flash-high","source_effort":null,"source_model_version":"gemini-3.8-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"4f3bf61e93663a340c7327d10cf822d76f59e596b705dc5911fcda0e91cf97e7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1269.72,"score_display":"1269.72","source_stated_rank":31},"run_fingerprint":"48239a8f1f641e756dbe8da346391a67509012c8561a6b41584fada341ff3842","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1ae51adf7ed122fed6c66bde663645864636545e1b60224705a10961ef302e51","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":6.865141,"date_is_proxy":true,"latency_seconds":519.104,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.884},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.8-flash"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"cc1969bd2ced73cb37352a6d6e2060b61ccfb36e885fa785a89e6019e9b645f3","result":{"confidence_high":86.26364,"confidence_low":71.03836,"sample_count":null,"score":78.651,"score_display":"78.7","source_stated_rank":24},"run_fingerprint":"3bb19c73eea23fd0e6a60256e692baee4c642522bd5f85d83793ae424a135869","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e96796b427d6557a779ba8a85347e474210ab5594bf810c2340f5e77d1344b14","metric_details":{"license":"Proprietary","variance":16.1873905395579},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3.8-flash-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bcab78acf579dfa16e885077d7d9e4e3316543af3c4aef5c7f8bc2a9bcbf0e67","result":{"confidence_high":1590.5681594793118,"confidence_low":1574.7968952088474,"sample_count":8524,"score":1582.6825273440797,"score_display":"1583","source_stated_rank":29},"run_fingerprint":"5ed9ade46cc264043393785815ee67e87e8283378116d73cbd9d06a71552a522","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4f1b7ffe8b756482fcac1d48361b08085dfd013b3251f5775c6c8b7c791112c1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.8-flash_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"df0b35a71b4e0429a625de4a25723505c8701938da209f4e3cd1b231289b5e9d","result":{"confidence_high":1580.25,"confidence_low":1555.5,"sample_count":2732,"score":1567.88,"score_display":"1567.88","source_stated_rank":27},"run_fingerprint":"24adada076e2ff8eace4deec4f0f0858b22dbfe87a4cf6d2c5ca74e3b0994e0d","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"85d093b30e6e39097b693fa7b3d122ed2845f586ad9dd8390e18040dbe7cc0d3","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.544972,"date_is_proxy":true,"latency_seconds":554.797,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.375},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.8-flash"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"a537c465a98a0ee69e5961e6a336b85b6362cbc3879ed3a6d9c1a182fe001cfc","result":{"confidence_high":82.008,"confidence_low":80.538,"sample_count":null,"score":81.273,"score_display":"81.3","source_stated_rank":8},"run_fingerprint":"13e055c8b839a94a72629325522cd1b40e41ae2bd47f80bff02659183188f089","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":32,"at":"2026-09-02T19:52:33Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T19:52:33Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4dd6f7b7783a7ed94435df84cea69b208fbcd268b90b81927bddb703d5710738","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.8 Flash (high) · terminalbenchV21"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-8-flash","upstream_model_label":"Gemini 3.8 Flash (high)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"8b76e3f306f29c5deefb51b32ddbc633abbad8b2a5118e9ba46627a5bb206255","result":{"confidence_high":92.956555,"confidence_low":79.20937,"sample_count":89,"score":87.640449,"score_display":"87.6","source_stated_rank":null},"run_fingerprint":"9c30d100700eae1ef41394d95155d6021a9632657a73b54cf84c3226c57cd1ba","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (high) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":32,"at":"2026-09-02T19:52:33Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T19:52:33Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5413f9b120ba4879b7d414d75bca7509913f50917f1413f3d80bb30078251ff2","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.8 Flash (medium) · terminalbenchV21"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-8-flash-medium","upstream_model_label":"Gemini 3.8 Flash (medium)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"0244a4853ae9a2e7f5e5fd42e05a6c835add04be8a235df56f5ae0da22fe5c13","result":{"confidence_high":90.100094,"confidence_low":74.885142,"sample_count":89,"score":83.895131,"score_display":"83.9","source_stated_rank":null},"run_fingerprint":"7defff24e2f1278390e53bd3d749c32eb6704ec5b16c7e8f75d2b4031043911b","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (medium) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":32,"at":"2026-09-02T19:52:33Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T19:52:33Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"91d8a362b931f8b308a30256ebcd670e7f702f50e60ebe513db026b6cb94763b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.8 Flash (low) · terminalbenchV21"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-8-flash-low","upstream_model_label":"Gemini 3.8 Flash (low)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"6c5464957f84c44032d98822061dd74f4783ac324270a7f0b5e18852d1d0ce9a","result":{"confidence_high":89.511838,"confidence_low":74.03726,"sample_count":89,"score":83.146067,"score_display":"83.1","source_stated_rank":null},"run_fingerprint":"5089942817859968b04137d19eed55cb36dd4f5734f5c3269e44b7bed9c3908e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (low) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6bd685a3e507031b119c9e85eef2dce1e9911bcf65b0550a2669545edabf0606","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (High) · tauBanking"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash","upstream_model_label":"Gemini 3.8 Flash (High)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"b0433beb00c6eea6d8d98cc27e518d689b0ab24c1936d1a6423ab8f05fee01d3","result":{"confidence_high":54.851888,"confidence_low":35.429902,"sample_count":97,"score":44.948454,"score_display":"44.9","source_stated_rank":null},"run_fingerprint":"0815b8640c124b11410fe031a5b41b15f5fb27d8a09295e9b8afaad13c2d2e54","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (High) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"83badb65c07a9ebfbc5714bbdc7d09bae7dd7cced2dbeae8fa3a43c5a78c3e8e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · tauBanking"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-medium","upstream_model_label":"Gemini 3.8 Flash (Medium)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"68226acf5177ee2cd9f2caa97a56f65bc68a97875f64799b036e941e897b54af","result":{"confidence_high":55.659638,"confidence_low":36.208798,"sample_count":97,"score":45.773196,"score_display":"45.8","source_stated_rank":null},"run_fingerprint":"45c542fcb210da1d5d2b1b309090dcc0f0886a3283d62e0b0e18a8d6bc51b2ee","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"724043e6f0de73f75fdddced8637ba4083450a4d991d6e5ef0a694103d2ba05b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · tauBanking"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-8-flash-low","upstream_model_label":"Gemini 3.8 Flash (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"3961ec7f0ad3361c5834ca291b58d78ae2e1e77edc60e295c7e7c9e25cd6870f","result":{"confidence_high":43.049664,"confidence_low":24.622407,"sample_count":97,"score":33.195876,"score_display":"33.2","source_stated_rank":null},"run_fingerprint":"f7aeb86ee90cc08415389a64a6ac8277164d5c2fa7efcf1327c68246bfb7de1f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ddd50a59d9c543af7739ffe0715c10bc1dfb7eacdbefede542ce530acb43dd83","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.746","mean_standard_error":"4.3","model_release_date":"2026-09-02","notes":null,"standard_error_points":530.0,"upstream_source_url":null},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash","source_effort":null,"source_model_version":"gemini-3.8-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"1f2e9ee59eef4164ad2c93a5b0c1ccef3f48071467ffbf66a1b4cbf431a3bc74","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":64.3,"score_display":"64.3","source_stated_rank":8},"run_fingerprint":"b4475e0476829f19331f34c2b26f2c47ab3ed5650ccebdeb540a72d4353c068d","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_51f85ec2f1c37d020adb2824","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d6c3ee05ac9daa0d8ce2f0f72232b53b6081d727463463e6800b747125b630d5","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":34396},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Gemini 3.8 Flash (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4ff4bab08ada6501f74a9edf4dc13246111d1405329d9595eb35ae691fd04a67","result":{"confidence_high":0.038071944536798685,"confidence_low":0.021228913793662735,"sample_count":4279699,"score":0.02965042916523071,"score_display":"0.0297","source_stated_rank":21},"run_fingerprint":"e4eef469a392c5c28a4d05cbce7ae9495e6233b3eeaaee41ddcbe8ef7277d987","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8f4369c92153a7efb62aacd816f4c4cce76e52ca03e04707e4a704af1c4d2456","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.8 Flash","source_effort":null,"source_model_version":"gemini-3.8-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"97b9872436953baf92d74f92534fe81f89f3cce8d3f8233af47bbf14ef27df99","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5093.793333333332,"score_display":"5093.79","source_stated_rank":27},"run_fingerprint":"e2daf8d99c8933c76a468d50263238fc5d6dc453bdb15c8d4e561ae92f286143","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cdf54735b765147ceab18bce","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6265941bbf9310b5739fe6a9906f3af725897b0809f7520125d599c7312261ee","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Gemini 3.8 Flash (High) · gdpval"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-8-flash","upstream_model_label":"Gemini 3.8 Flash (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"ed71711c53e2361eea53eb88b082b5f9147d5d5b6432fbd5aef6b8602a3e8002","result":{"confidence_high":1437.4,"confidence_low":1386.5,"sample_count":220,"score":1411.95,"score_display":"1412","source_stated_rank":null},"run_fingerprint":"20a73cbdf8871260f3e331cab2976a6c61ce2edcc68da64805d3b196e560e522","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_6307c6a5c62851b4f5c8d080","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aa093562313bf0d0cc77fa4bfcdd9925247c0dd916269b6fc9407295cab6d72c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · gdpval"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-8-flash-medium","upstream_model_label":"Gemini 3.8 Flash (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c73dba19447bd8358dfcf1edfe2582cb1bd8f8a2d48542324da893a07b8aa980","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1408.58,"score_display":"1409","source_stated_rank":null},"run_fingerprint":"2f5cf0de4e8522c4c944539980a6a6f0f9b7d23ba9e94f43969e417865cb52a9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_eb17f29c7f8397dab70e09d7","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"763d29e4a0698533c72011a5ff0360abce49b590d41bcaded8709b0011aab3cc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · gdpval"},"model":{"id":"google/gemini-3-8-flash","name":"Gemini 3.8 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-8-flash-low","upstream_model_label":"Gemini 3.8 Flash (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"db698a8668f2c6581dcbd2b3b9a2e31ea705ea516c0b9d4a0cca68e5159bc7fa","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1294.36,"score_display":"1294","source_stated_rank":null},"run_fingerprint":"c282e78fa8d32ca77d71c92d03cfe59ae7d2ec59bcf2a01566352e300e8873f9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.8 Flash (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d34f161872701271f8e1f8ac","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"712dc22c5bb50519d25c08eda8173d0acc96d8e54e2be2f245f962a852e857cc","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-03-05"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.4; model release: 2026-03-05","source_accessibility":"API access","source_model_name":"GPT-5.4","source_model_release_date":"2026-03-05","source_model_versions":["gpt-5.4-2026-03-05_high","gpt-5.4-2026-03-05_low","gpt-5.4-2026-03-05_medium","gpt-5.4-2026-03-05_none","gpt-5.4-2026-03-05_unknown","gpt-5.4-2026-03-05_xhigh"],"source_reasoning_efforts":["off","low","medium","high","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e8034c54bdbb1611a31443775bb9964e2ac497ae6956da52daa43a260a3daf61","result":{"confidence_high":159.1,"confidence_low":154.91,"sample_count":null,"score":156.91,"score_display":"156.91","source_stated_rank":16},"run_fingerprint":"629f2246015cc3954cca8aa4b2be669a4a68779bbb39ae0675553508ddfb0895","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54ae75f31e2b8962dbf99cae","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f6849bba634a25e5bdd1bd93026df553a5b72c98af3548f7b40ad6366cd80e61","metric_details":{"license":"Proprietary","variance":3.706462012373595},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1479.0163861126327,"confidence_low":1471.4696726051516,"sample_count":64642,"score":1475.2430293588923,"score_display":"1475","source_stated_rank":34},"run_fingerprint":"70c9021ec76846d4e4b876e36ca8ab9312e28f67c2ce3960f83cff3b9d7623ad","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8d8567e6b071afe3a604c7960a4d1c77084d820a62bd54be0e99410685744700","metric_details":{"license":"Proprietary","variance":3.5603147843514225},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1468.4974165328588,"confidence_low":1461.1009842489007,"sample_count":67707,"score":1464.7992003908798,"score_display":"1465","source_stated_rank":53},"run_fingerprint":"39470309ece409faec4dc2d6d19f01ce42b7dbdbb1793e5d5b4d16dff77f5263","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_24c8776cdb248349a103a879","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ca6aff9055bc18dcc60b98ef38032a9e7e8cb6a6cb6e3fb01d532204c493d033","metric_details":{"category_scores":{"Agentic Coding":53.83833333333333,"Coding":77.5415,"Data Analysis":79.31333333333333,"IF":70.21674999999999,"Language":82.63366666666667,"Mathematics":94.148,"Reasoning":88.1155}},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.97244047619049,"score_display":"77.97","source_stated_rank":19},"run_fingerprint":"debde403fadd1702190162b16e066c052499b6381f1efbb10855d3a71490a370","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":208,"at":"2026-03-10T21:09:26Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-10T21:09:26Z","status":"stale"},"measurement_id":"da927d6521383af1bfe3103c89a262dc186ae0f12deb624750da87149bd8aa24","metric_details":{"calibration_error":42.0,"confidence_interval_half_width":1.88,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=gpt-5.4-2026-03-05 (xhigh thinking); createdAt=2026-03-10T21:09:26Z","source_row_created_at":"2026-03-10T21:09:26Z"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"gpt-5.4-2026-03-05 (xhigh thinking)","source_reasoning_label":"xhigh","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6fcb44592398eda2b6c34d8006614d73ea5ec5b19964bf7e3c0e28c9d4464fe8","result":{"confidence_high":38.120000000000005,"confidence_low":34.36,"sample_count":2500,"score":36.24,"score_display":"36.24","source_stated_rank":6},"run_fingerprint":"06d59e6b2dd955e63bd4211d2acec7a1702613807b85906965e55645de660390","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=gpt-5.4-2026-03-05 (xhigh thinking); createdAt=2026-03-10T21:09:26Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":214,"at":"2026-03-05","basis":"evaluation_at","evaluated_at":"2026-03-05","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"d1d5cd73d3f258c37dd482647724b00778c8fb090816052a053c40c04152ea10","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/openai-gpt-5-4","source_model":"GPT-5.4"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5f574755273dec69223ea106198219ab85de60f2ba6ba61150a9408295e331a1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.28,"score_display":"40.28","source_stated_rank":13},"run_fingerprint":"333eb37e7e98461a6614b815127642e41a760bcee7894e8238993bb92a9662ea","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4f103a84cd706b6426b9557c","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:28:16.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:28:16.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ad868a46b48d600844cc490f3c70bef4693b469c9b3491c4ef1799c6074266e6","metric_details":{"best_score_across_scorers":"0.451","model_release_date":"2026-03-05","stderr":1.5743152379585443},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"AzCkm282PLhDdLJ8DnAQQc","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAzCkm282PLhDdLJ8DnAQQc.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/AzCkm282PLhDdLJ8DnAQQc.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"af3b5b3df99ddad25db0a7afaba9b2d3052c3e051686d2c52d261ca34bed3a0c","result":{"confidence_high":48.18565786639875,"confidence_low":42.014342133601254,"sample_count":1000,"score":45.1,"score_display":"45.1","source_stated_rank":39},"run_fingerprint":"bdf90508dfdd76d105259ef8079ec4e886498245e31d49317a848b551133473b","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":209,"at":"2026-03-10T00:00:00Z","basis":"evaluation_at","evaluated_at":"2026-03-10T00:00:00Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"beaafde1764302b2835ca41d8c8e90d6c37ee6ba9bd67474aecf5d20b7125f26","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-5.4-2026-03-05","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8c61b7b362b968f1153894ee150b25c2f09d305edfdd3c49c85935e04c04455e","result":{"confidence_high":33.49488405884845,"confidence_low":27.40963855421687,"sample_count":null,"score":30.45226130653266,"score_display":"30.5","source_stated_rank":25},"run_fingerprint":"94b239199c0e85ca297d984b580231430bab2ee54260f98b98f48f561df63166","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_415679f326b527dc324429c9","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f678405b490174c899ddb0861913c67edc6f0d8c3c9078ef8d3ed4c40fde7eaa","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.20934,"date_is_proxy":true,"latency_seconds":202.953,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.908},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":91.414,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"xhigh","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-2026-03-05","zero_shot_accuracy":91.919},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"b4317e19cc0a53ccc68426cd6d5f63c2f030f5492bce91764b7a89617e8edb0d","result":{"confidence_high":95.40567999999999,"confidence_low":87.92632,"sample_count":396,"score":91.666,"score_display":"91.67","source_stated_rank":18},"run_fingerprint":"232b4320b1eb294efb03c2f228a8a991222a3bc9f0f1004bc5863649f3026eeb","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":213,"at":"2026-03-06T02:57:02.864Z","basis":"evaluation_started_at","evaluated_at":"2026-03-06T02:57:02.864Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"829e54e86f986f4b08da2aebb5300731acd21c7c80a9c8133404f77be257e938","metric_details":{"best_score_across_scorers":"0.933","model_release_date":"2026-03-05","stderr":1.7999999999999998},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"fFatyce8UvpN7ZivdmrhAy","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfFatyce8UvpN7ZivdmrhAy.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/fFatyce8UvpN7ZivdmrhAy.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"327273f02733827f7a2c8de8cf61f3e66bd847b8158584f196cbf346e7809f2a","result":{"confidence_high":96.82800000000002,"confidence_low":89.772,"sample_count":null,"score":93.30000000000001,"score_display":"93.3","source_stated_rank":18},"run_fingerprint":"eab73cdbcfeacef4ede73cb265fa466de2a6a428255bbc5bfea6d6296d6fa23d","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":82,"at":"2026-07-15T03:14:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:50.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6abb7b1de835a655167ba201c83fccee2c8d2606b38d9a375261b74e5997bfd5","metric_details":{"best_score_across_scorers":"0.898989898989899","model_release_date":"2026-03-05","stderr":2.1469735576055355},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FbhcT8NnJUntcyNGubJXY6","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFbhcT8NnJUntcyNGubJXY6.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/FbhcT8NnJUntcyNGubJXY6.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6536e34f8e61dfaffffbb801ccd7c018df9682427eee759c3252f440f1553680","result":{"confidence_high":94.10705807189674,"confidence_low":85.69092172608305,"sample_count":null,"score":89.8989898989899,"score_display":"89.9","source_stated_rank":46},"run_fingerprint":"28860a5ddd2462cf4f051cf7da3397337563768aa2c5073f85bdf76bdd7353b6","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":82,"at":"2026-07-15T03:14:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:58.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"995451ce43e1adfb1d717ef17385bb7fcadf58dff1c65874915520ec6dfb9c16","metric_details":{"best_score_across_scorers":"0.8888888888888888","model_release_date":"2026-03-05","stderr":2.239078763821682},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ManJk43Kf9VuXowNqoDrov","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FManJk43Kf9VuXowNqoDrov.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ManJk43Kf9VuXowNqoDrov.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"88afabf513e62fd7dac9e9da03d31dc0a91853df18297974ba3d5ab7c12e22a9","result":{"confidence_high":93.27748326597938,"confidence_low":84.50029451179839,"sample_count":null,"score":88.88888888888889,"score_display":"88.9","source_stated_rank":53},"run_fingerprint":"a70a38147897a2dc2125d79b183e773a5a86c5472354cf1b750392be8d997db2","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":82,"at":"2026-07-15T03:14:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:49.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6c3d95a0c35309b4510662ba814f809601d31bd7e7af86988888b4cdb3be9961","metric_details":{"best_score_across_scorers":"0.8484848484848485","model_release_date":"2026-03-05","stderr":2.554565042660364},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"aYgDc2iSDpmHkQfu7YjCHc","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaYgDc2iSDpmHkQfu7YjCHc.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/aYgDc2iSDpmHkQfu7YjCHc.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"2411406a3388ccf79a91cd463b1e057c8c3266e9fc088e7b71ed6e11c1dba2e5","result":{"confidence_high":89.85543233209916,"confidence_low":79.84153736487053,"sample_count":null,"score":84.84848484848484,"score_display":"84.8","source_stated_rank":92},"run_fingerprint":"49f3b4c17593f4daaa057d7c5ccf204b84476ae7454e33239b7b747566b757b1","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_f481293cab8e00936a97d0f2","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":82,"at":"2026-07-15T03:14:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:58.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ff26b4629028b516e024c452da66b349ad64ccb31cb6a80e6a8995c4a06c0d8e","metric_details":{"best_score_across_scorers":"0.7474747474747475","model_release_date":"2026-03-05","stderr":3.095405547036587},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LaBuzZExRqocD4FET6yx26","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLaBuzZExRqocD4FET6yx26.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/LaBuzZExRqocD4FET6yx26.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"4837bc6b5e363e14499d3c9db4dd95a8b642f20b1672122b93daff98134c99a0","result":{"confidence_high":80.81446961966647,"confidence_low":68.68047987528304,"sample_count":null,"score":74.74747474747475,"score_display":"74.7","source_stated_rank":159},"run_fingerprint":"abec1fe27e64016efee4934faa73a064276879ae82603666e5490cf70b5fb29e","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_da6182d6c493323255d5f78a","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b63237404ce3faefafda7165aac5a8e93cf671413e9df653feb8439843894abd","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.050888,"date_is_proxy":true,"latency_seconds":28.875,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.421},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-2026-03-05"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"eb938b892d499ceb04b9d11e71d37835853748fcf12ae2bcec97f18cd2cf0365","result":{"confidence_high":88.30716,"confidence_low":86.65684,"sample_count":null,"score":87.482,"score_display":"87.5","source_stated_rank":27},"run_fingerprint":"f3677672ef072d43e51e1f17267360159f5e109959205fbc7763e49e94202b2c","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":213,"at":"2026-03-06T03:24:03.099Z","basis":"evaluation_started_at","evaluated_at":"2026-03-06T03:24:03.099Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0f45bb4aa14fe247620af2a380f46dc08971e2818f92699471b02ed342fc3b22","metric_details":{"best_score_across_scorers":"0.953","model_release_date":"2026-03-05","stderr":3.2},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"HVsPchr34pWYHNuv5sKYyK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHVsPchr34pWYHNuv5sKYyK.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/HVsPchr34pWYHNuv5sKYyK.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5700dff2b509c55ed69571444ee543107a87208784ee03e6590b0fbaa8412924","result":{"confidence_high":100.0,"confidence_low":89.02799999999999,"sample_count":45,"score":95.3,"score_display":"95.3","source_stated_rank":46},"run_fingerprint":"0533e7ffd1c7803b06d684fa02080efe291f112e1c648b62829984e7ac93ae00","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:50.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ff757776850f85093a0d6679be31fd50f8ef3712bf7423edf838e188612f4501","metric_details":{"best_score_across_scorers":"0.9777777777777777","model_release_date":"2026-03-05","stderr":2.2222222222222223},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"DpPahiEawokzgidE3cNZbS","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDpPahiEawokzgidE3cNZbS.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/DpPahiEawokzgidE3cNZbS.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"37f7370cbd2dbc09571ef0b7fc4f620012feee7617a024935f662204e34d8fa8","result":{"confidence_high":100.0,"confidence_low":93.42222222222222,"sample_count":45,"score":97.77777777777777,"score_display":"97.8","source_stated_rank":25},"run_fingerprint":"92b691f46e621f9add7f7b5e231c848d9b6ef2c20bc8795a61182c8041385898","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:58.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a0b4a5507fafacb943b39afbdc7b76a1fe7271150d01e99c81e647f25cb3ead0","metric_details":{"best_score_across_scorers":"0.9555555555555556","model_release_date":"2026-03-05","stderr":3.1067790907534736},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"eUJNMJSdLnJWNgRY8D2rYw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeUJNMJSdLnJWNgRY8D2rYw.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/eUJNMJSdLnJWNgRY8D2rYw.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"899efbcbb33860009b73ddc830f259ea3b8202f55fee46e88beb50264cda1056","result":{"confidence_high":100.0,"confidence_low":89.46626853767874,"sample_count":45,"score":95.55555555555556,"score_display":"95.6","source_stated_rank":38},"run_fingerprint":"ff9fd4575296d3913b883642a6b98a0b0f6634771d9d0ff71262fb0beb6e6b8e","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:49.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ebf41d657c9d57ce95f26e2d4b109276f36391a7bebe183714015e52a05915f2","metric_details":{"best_score_across_scorers":"0.8444444444444444","model_release_date":"2026-03-05","stderr":5.463890236888291},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bezRyWjXxqHnQeiL4MkSfZ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbezRyWjXxqHnQeiL4MkSfZ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bezRyWjXxqHnQeiL4MkSfZ.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"678cb47da5384a66d64e53044ec34271aa4563d0395ddc8ce1052b986f96e854","result":{"confidence_high":95.15366930874549,"confidence_low":73.7352195801434,"sample_count":45,"score":84.44444444444444,"score_display":"84.4","source_stated_rank":93},"run_fingerprint":"b0c94752c445947ddeb060cae8e0f639ee3905e4134acf7a6327c00f0665fc82","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_f481293cab8e00936a97d0f2","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:58.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d9cbc48505288747306d8400ecf6dd8c102651e95735bad70366cb8cb326813e","metric_details":{"best_score_across_scorers":"0.5777777777777777","model_release_date":"2026-03-05","stderr":7.446027270295804},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"fjxs2EWts9GjpuBtFD5BN3","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Ffjxs2EWts9GjpuBtFD5BN3.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/fjxs2EWts9GjpuBtFD5BN3.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9fe1789101b3b54a2449506a30d214be5b05ec3b0227bcaec5eb2aa21bb0f6c4","result":{"confidence_high":72.37199122755754,"confidence_low":43.183564327998,"sample_count":45,"score":57.77777777777777,"score_display":"57.8","source_stated_rank":169},"run_fingerprint":"6f62c517da523717d80b2f4be739e38386992087e4954792f91b08facf33ecb8","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_da6182d6c493323255d5f78a","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T17:55:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T17:55:45.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"04260d1c2acf48bea4ad7bc10e074665b96e728e14ee9cdff064d28be22b6f76","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.7859649122807018","leaderboard_private_problem_count":285,"model_release_date":"2026-03-05","public_example_count":10,"stderr":2.4338000553262393},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"PaYEQ37A2n6giYBwZGBnXY","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPaYEQ37A2n6giYBwZGBnXY.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PaYEQ37A2n6giYBwZGBnXY.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 (xhigh)","thinking":null,"upstream_model_id":"gpt-5.4-2026-03-05_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"2a49bf01a775f4a733f04972aced6d25537229525897b14e8d8946d46425c6e7","result":{"confidence_high":83.36673933650961,"confidence_low":73.82624311963075,"sample_count":285,"score":78.59649122807018,"score_display":"78.6","source_stated_rank":17},"run_fingerprint":"b75b94d004be0802b73a7e8c5636238c1607ae3b7a9c7b2844a779cba589d4a4","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"12f4b4b4969d166db90bb32b6de0d3f4af20546697f2b89a2e24ff6ce5e7b197","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.52,"model_release_date":"2026-03-04T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4","model_type":"CoT","upstream_model_id":"gpt-5-4-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b78e285845cfd461df605335673828086cb4d79a24f9cc35618eb474ae942716","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.95,"score_display":"74","source_stated_rank":48},"run_fingerprint":"2abc3a65a86a1d5e462f0f2f6b930d6ebe4cbc6dd9a491d30f0ca775ed0aa824","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aaf4395f239f2de8d4aa6a932f29142006ab8229c14f2d3206db52dcaeef2320","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.02,"model_release_date":"2026-03-04T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4","model_type":"CoT","upstream_model_id":"gpt-5-4-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e52164abbde004f067f3ac44afbd32eeaea925f8b9ebb6d05d5da79c5d9137e0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.5,"score_display":"68","source_stated_rank":59},"run_fingerprint":"519961005376a0a71ef90d0949fb0b926e91b61f1b2eae54d79beade075c980f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"976968bf760d3fa68b0b4553b8665327cdd6dcecd2b4b8c2a17fb6e98fafa066","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.68,"model_release_date":"2026-03-04T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4","model_type":"CoT","upstream_model_id":"gpt-5-4-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"df66a30a909cdba90740562770e65e636acadbc9cae0341fb857a7045fed7fad","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.42,"score_display":"55","source_stated_rank":87},"run_fingerprint":"851cc5f505daa5afebf4c0f74e04aba4a53f0b23132ee055c28bc0e3cba76941","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8060addeeec0f4c109f7ad95665fb5dab407d8493e67fe3dfaa8360a545ee5ac","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.27,"model_release_date":"2026-03-04T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4","model_type":"CoT","upstream_model_id":"gpt-5-4-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"be85d7c715b6d07f8ccf4344a2a34505aa48e7baf34d5df68a7dbc04bedff7fe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.17,"score_display":"29","source_stated_rank":121},"run_fingerprint":"a0e92376fd3be5e20beb92a83b33a518d4348ab6091e704a297e46700243bfcf","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_f481293cab8e00936a97d0f2","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"285b24d3fff8364e467fcf28e6d4d22ab196d25a9624b6cce26be084e99ed537","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.52","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (xHigh)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"811cf8a247a0715019c62adfd455ad294cafa440d4c3a65eac679ca76605c011","result":{"confidence_high":78.214171,"confidence_low":69.18008,"sample_count":360,"score":73.95,"score_display":"74.0","source_stated_rank":51},"run_fingerprint":"880d1d683bb04b26768260599f7eb34f60a697fd2db4c5b51caf69f29b489585","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"04dff08a6c0f881b0463656f13b114ae838979a47e5af9f5cca7cceb3a16ef04","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (XHigh)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"2b77e024c41f17e22d0273e631e4f63f27f76e896a12712d66a7cf844b3f097a","result":{"confidence_high":78.214171,"confidence_low":69.18008,"sample_count":360,"score":73.95,"score_display":"74.0","source_stated_rank":51},"run_fingerprint":"7546fa6df595275dfa009376e82d3ff9bdf5752af92dc13142dc7a9de9c7fe9c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1f2c3515dcfecdd0fc609387e1a7e615cae5361ae27db2dab3f6db4617dde1a2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.02","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (High)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"8460362f3eb7f1bdebb6a7691dd2733e03da5d2079a4713c79366aa704fdd0c1","result":{"confidence_high":72.131525,"confidence_low":62.49893,"sample_count":360,"score":67.5,"score_display":"67.5","source_stated_rank":63},"run_fingerprint":"8eb09601559883220e5005d9f3a48d17429e37110ccd22fc7d1445344c9e030f","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"975a82b52abb15e84d071a959f094cbbd71c04953921140843bc33c09ffa2e05","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.68","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (Medium)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"124d0419e93a62ec134b2e5b4278b71fd85b787b3e89fa6948251974af82d5ad","result":{"confidence_high":60.470533,"confidence_low":50.255014,"sample_count":360,"score":55.42,"score_display":"55.4","source_stated_rank":89},"run_fingerprint":"672848aa42a0a55f9fe80f1a895a0e233f4341f03f4e951ffe880ec497420810","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c567aac1a0bcd65671dd371cd7d38ccd1fb6373ef3fa5c269f577ca2b355b8b9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.27","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (Low)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"289580d337535db929849b15c1970544f43ec6bb2a8993b096e98b77c0bc2d8c","result":{"confidence_high":34.065751,"confidence_low":24.714113,"sample_count":360,"score":29.17,"score_display":"29.2","source_stated_rank":120},"run_fingerprint":"896d690396d3fded3c1a022d14863f526d31eeab3cc23c0f1c94318eba1dc386","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_f481293cab8e00936a97d0f2","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T17:55:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T17:55:45.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"032279415fecd8388bb1da56530a32b1bd8433814cdc376882c6ea2273fb19e8","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.49","leaderboard_private_problem_count":41,"model_release_date":"2026-03-05","public_example_count":2,"stderr":8.0},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"doiQDghqnLzbE8CFijLNQJ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdoiQDghqnLzbE8CFijLNQJ.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/doiQDghqnLzbE8CFijLNQJ.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 (xhigh)","thinking":null,"upstream_model_id":"gpt-5.4-2026-03-05_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"27b31f681b895e22783a77eed993fda0f1bbc4edf584eaf1a399ee47123a3269","result":{"confidence_high":64.68,"confidence_low":33.32,"sample_count":41,"score":49.0,"score_display":"49.0","source_stated_rank":22},"run_fingerprint":"1fb448477bbb5124ad040a38c8a6e680c00476259a8a900dc97c83ced8daa76a","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9d67a5c235d25455c5e38c40ca80d27920e957f6d3e0ffaea6b80c4978a905b8","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.047083,"date_is_proxy":true,"latency_seconds":65.565,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.795},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-2026-03-05"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a4ba51146227904e115d28c39e40bb84142588638113232b70e271427277a220","result":{"confidence_high":89.0722,"confidence_low":85.9558,"sample_count":null,"score":87.514,"score_display":"87.5","source_stated_rank":14},"run_fingerprint":"8debac41e9e446beec9f78a31aaf6feac8554e7e5693f907898ab08600b1302b","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6991732b291d60052533a02946f05e53da94605c7ff72177480a416b329d41a8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (xhigh)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"ea2b310e4119a99d31b5c3ff9c76a1ef091cea8acb261867adc06a3b38c981c0","result":{"confidence_high":34.484932,"confidence_low":15.10002,"sample_count":71,"score":23.42857143,"score_display":"23.4","source_stated_rank":43},"run_fingerprint":"9aef6f74eada5321c3cfd37881b999c2cd103824e28db4f1a3f736592362d2e2","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":208,"at":"2026-03-11T10:19:58.027Z","basis":"evaluation_started_at","evaluated_at":"2026-03-11T10:19:58.027Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"627992c0a9679b6c4c91e9296db60b0c2d1cfad8bd9b5bc20504b33bdf5e0501","metric_details":{"best_score_across_scorers":"0.44","model_release_date":"2026-03-05","stderr":5.0},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"RjvLecpyS4VVkJPSXhk8Ju","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRjvLecpyS4VVkJPSXhk8Ju.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/RjvLecpyS4VVkJPSXhk8Ju.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8330aac0bb5b5bb0781865966f937cb5be1dc9d79735fe50adac0bf945abf236","result":{"confidence_high":53.8,"confidence_low":34.2,"sample_count":100,"score":44.0,"score_display":"44.0","source_stated_rank":18},"run_fingerprint":"464831a7ad6613a678c89d408c75c048275ff845353ca63204eae62661dafc1e","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:50.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"402d87bb728df298bdb00a58bac2589ffb17c0b8e3b9db1107fa46642a4fb8c0","metric_details":{"best_score_across_scorers":"0.38","model_release_date":"2026-03-05","stderr":4.878317312145634},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"fQJLivo4QNbmQqnhyoXLav","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfQJLivo4QNbmQqnhyoXLav.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/fQJLivo4QNbmQqnhyoXLav.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6fd7109500de24552a2f7405ee591b75d1a62bc9dacc34aa24d7b54cb58c6ec4","result":{"confidence_high":47.56150193180544,"confidence_low":28.43849806819456,"sample_count":100,"score":38.0,"score_display":"38.0","source_stated_rank":32},"run_fingerprint":"27a363b933054edee5447ece2d5225df39e4c75bdbfd80545de269380b38e12c","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:58.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7c89a36821863046ef38fe05800480597f375d9ee13d4bd28b0c6565e1bc62e5","metric_details":{"best_score_across_scorers":"0.38","model_release_date":"2026-03-05","stderr":4.878317312145633},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"n9rnjVzibawgtMunvu5yEp","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fn9rnjVzibawgtMunvu5yEp.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/n9rnjVzibawgtMunvu5yEp.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f5d37b8028e37df6291db20c71060be990a995017d583706b675b65830d380b2","result":{"confidence_high":47.56150193180544,"confidence_low":28.438498068194562,"sample_count":100,"score":38.0,"score_display":"38.0","source_stated_rank":32},"run_fingerprint":"4c1a0158d5012a27412cc8e08f80a7db76fe030f6863c853d3be673004861d2a","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:49.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2b2f3955ff46b5458a2b85d5dee10e768dba2ed881fbdd0361a59d75a1928d5c","metric_details":{"best_score_across_scorers":"0.2","model_release_date":"2026-03-05","stderr":4.020151261036848},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"5GLQdqyuDzgBmeZffzuwQx","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5GLQdqyuDzgBmeZffzuwQx.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/5GLQdqyuDzgBmeZffzuwQx.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f9782788ccb0741dffd7c4688700165aa8f8e2ae5938f81e517db5e2ed80200e","result":{"confidence_high":27.879496471632223,"confidence_low":12.120503528367777,"sample_count":100,"score":20.0,"score_display":"20.0","source_stated_rank":87},"run_fingerprint":"f484ceaaa25d65031a452cf59d63863d285618a4ac23e21f64f58973b030a8b1","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_f481293cab8e00936a97d0f2","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:58.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"346f9cef5634998846fa515d0905522c09766455a7de39d49f8bab44d1226b10","metric_details":{"best_score_across_scorers":"0.05","model_release_date":"2026-03-05","stderr":2.1904291355759034},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6M8MmmtFv82WVQzUasKhYT","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6M8MmmtFv82WVQzUasKhYT.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/6M8MmmtFv82WVQzUasKhYT.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"79e45e040dd9c2da093f758ee0a9e931beffcba94a73ceab7ddf0e0e535a5b9f","result":{"confidence_high":9.293241105728772,"confidence_low":0.7067588942712293,"sample_count":100,"score":5.0,"score_display":"5.0","source_stated_rank":161},"run_fingerprint":"d8600d6a1a8439bdfe11e8a425578b19a5773404f10f227c4940c73b7bd48988","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_da6182d6c493323255d5f78a","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9489fb19ec381a070ed1fc61c97aa3742cd0e896a9cb2d4f97b4156cfa52423f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.62","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (xHigh)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"69829db519a75c0e24f5b48887b1bc469c3e35ca16a2b02491018075b34490b7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.67000000000002,"score_display":"93.7","source_stated_rank":46},"run_fingerprint":"cdf129b27f91d8dd8e74a682ffc227311e5f31322d58d18fe99511902f51790a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6cb1e1a3522c87ca23877ee812a71cc286c5e7de72f50d57100347f5dacf6ad3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (XHigh)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c3c8ff00d201eade6b0ab28ef1ae2c3e41e5937f49408b614729c23c27a9be71","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.67000000000002,"score_display":"93.7","source_stated_rank":46},"run_fingerprint":"676a1cfdeb178b83f3ee0eb2862beaba131a3bffe37d92d516712b339f080697","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6c95f1e8458a42373d4300ebee23db511cceecefa1652f462019225840c0ac31","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.37","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (High)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"302e634ecd9ee9766a7c6b392624119c4c98c51da63c62eddbca0101e7cbb36f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.67,"score_display":"92.7","source_stated_rank":52},"run_fingerprint":"63468fc8b15e3091e79b0fe397fa2d90cef33f1bb1146243e7840d6679aff916","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bb8262ce12cd645563c4b92e1ac8b5b4bad0b5f3bee8de48f96e1990523fb799","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.25","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (Medium)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"591d95f1d0a2d22f7b7fedfbabcc137388e0ed78bdf27159076fec621e6ec463","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.17,"score_display":"86.2","source_stated_rank":96},"run_fingerprint":"caab896818007f5a46658348570bada2cfe0763a225246b07f8f060c14675bd3","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b57ceda32f64b628890fa7db936ced939a1c79c9176e5957387919c7e8a4ede1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.15","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (Low)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"7a57220094c4e47b09c37d9b311f8181b910b3387126dcfc1558287015c20017","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.17,"score_display":"68.2","source_stated_rank":137},"run_fingerprint":"f61ebe77aa3e39d80c88f71d8c22dad9850b859fa39ee5693810c7521420aa1c","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_f481293cab8e00936a97d0f2","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"afef50fbbd4c8999da1c6b33b22ef7436e2eaa883a359106c7cc7fb1c0050aa4","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.801134,"date_is_proxy":true,"latency_seconds":307.118,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.848},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-2026-03-05"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"70dbd34bed2173625ff18d1d427b2b4faf70794e1a8ce58ed504d51cee9f36cc","result":{"confidence_high":81.82208,"confidence_low":74.57792,"sample_count":500,"score":78.2,"score_display":"78.2","source_stated_rank":30},"run_fingerprint":"32b5c56f9a94b868ff8ccaa63fa13b893475128c90191f62292607bd8eb6873c","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":213,"at":"2026-03-06T11:07:43.396Z","basis":"evaluation_started_at","evaluated_at":"2026-03-06T11:07:43.396Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"cc17ad175af3eb6edf432b82d7025f45420abb1e7cd4a951e822a66e059e234d","metric_details":{"best_score_across_scorers":"0.768595041322314","model_release_date":"2026-03-05","stderr":1.9189407003691976},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"i9HTMbAxcwsrTf2w2FuJ6N","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fi9HTMbAxcwsrTf2w2FuJ6N.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/i9HTMbAxcwsrTf2w2FuJ6N.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"fe21a2d77859fb30565c633f3100c90c67771aa8cfc857c0d35a5633b6cc855c","result":{"confidence_high":80.62062790495503,"confidence_low":73.09838035950779,"sample_count":484,"score":76.85950413223141,"score_display":"76.9","source_stated_rank":8},"run_fingerprint":"ab2d71f6eb7a749f29539b79e9f7456634bb0f3ff0e4816c93477b6a70247253","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_84d7ad8aa122e93390d30e06","split_or_window":"2026-03-15/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-29T23:53:02Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"7eb56263d626318606faf06a0ed6b3f2a926d26b7699077886d1108a08226e9f","metric_details":{"comparison_warning":"Official post-release SWE-rebench window; use only for same-window comparisons and never mix it into the canonical rolling-window rank.","comparison_warning_code":"swe-rebench-post-release-window","model_release_date":"2026-03-05","pass_at_5":71.27659574468085,"potential_contamination":false,"sem":0.796097316334882},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-post-release-window","comparison_warning_code":"swe-rebench-post-release-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","supplements_range_key":"1772323200000:1778803200000","upstream_model_id":"gpt-5.4-2026-03-05-medium__tools","window_from":"2026-03-15","window_status":"supplemental-post-release","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"9ba14dbc07b4734d9269ca0619063b4d06c575561ea00192c192a1576772ae1e","result":{"confidence_high":57.517797548527014,"confidence_low":54.39709606849428,"sample_count":null,"score":55.957446808510646,"score_display":"55.96","source_stated_rank":null},"run_fingerprint":"1fd030adc5fef5cdf1b1088935b77a6b27fdde33f516138b63c8e85df83a81d1","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-post-release-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"41fda1e7c7c6a549acc20fac035d2cd02239c0873e4fa2b92ce373fde50ec443","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-03-05","pass_at_5":70.9090909090909,"potential_contamination":true,"sem":1.0204520145747118},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","potential_contamination"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"gpt-5.4-2026-03-05-medium__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"ce8174030f583c6075758ab72f84c5cfc1f787e782478ec97705b870f291415b","result":{"confidence_high":56.90917685765734,"confidence_low":52.90900496052447,"sample_count":null,"score":54.90909090909091,"score_display":"54.91","source_stated_rank":null},"run_fingerprint":"02e5d2157cfdead40490fcbf94f722e848997ea142f9a285ec4ba966452875bc","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"65041becdd328b9694ced5596e82faa38d8e071ea75c1559b7a4c40f589b0ddb","metric_details":{"ci_attempted":452,"ci_half_points":1.5021049567382834,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":234,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":70.47123893805309,"mean_cache_tokens":8613618.407079646,"mean_cost_usd":5.6524577245575225,"mean_duration_seconds":1368.707164818584,"mean_input_tokens":9418920.763274336,"mean_output_tokens":71408.87389380531,"median_agent_steps":63.0,"median_cost_usd":4.3528565,"median_duration_seconds":1209.517342,"median_input_tokens":6869950.5,"median_output_tokens":67233.5,"median_output_tokens_to_pass":64347.0,"median_peak_context_tokens":172734.5,"n_attempted":452,"n_passed":234,"n_tasks_attempted":113,"n_tasks_passed_any":88,"pass_at_4_points":77.87610619469027,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_4_xhigh","source_model_version":"gpt-5-4","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"a7d2915ef00e671de2b4cb127c262c7196f5469dafa2a20cfc59b38fd001234a","result":{"confidence_high":53.27201646116306,"confidence_low":50.2678065476865,"sample_count":452,"score":51.76991150442478,"score_display":"51.8","source_stated_rank":48},"run_fingerprint":"74b87fbe5cb9578320ea8f96d94326d160e719665b2a6ddbc54ac844344a2018","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4378cab7abe5bb5312d520ded5dfe523460c94d4a7f917addf82003aab57110e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"70.47123893805309","mean_cost_usd":"5.6524577245575225","mean_output_tokens":"71408.87389380531","model_release_date":"2026-03-05","notes":null,"pass_4":"0.7787610619469026","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-4 (xhigh)","source_effort":"xhigh","source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"901a187c6dfa4e5c0480a24b364256a2e568a3a800e9522008917e2941d4b13c","result":{"confidence_high":53.27201646116307,"confidence_low":50.2678065476865,"sample_count":113,"score":51.76991150442478,"score_display":"51.8","source_stated_rank":46},"run_fingerprint":"8c42a561154a89fcc349137725b945f77883dfe36bcc6fd642dd12cbffbdd946","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":179,"at":"2026-04-08T17:04:48Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-08T17:04:48Z","status":"stale"},"measurement_id":"c72951472dacf384b56cd1f7ebf1970cf98bfebc77724633856ed372bf622a84","metric_details":{"confidence_interval_half_width":3.56,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gpt-5.4 (xHigh)*; createdAt=2026-04-08T17:04:48Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2026-04-08T17:04:48Z"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":true,"cost_cap":"uncapped","protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"gpt-5.4 (xHigh)*","source_reasoning_label":"xhigh","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576","turn_limit":250},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"b943c0c9912e67509be831aee5509ed0b6ea3fa75d30f5be2a029514da247f14","result":{"confidence_high":62.660000000000004,"confidence_low":55.54,"sample_count":731,"score":59.1,"score_display":"59.1","source_stated_rank":1},"run_fingerprint":"177a553cf2688bfa40e4ca0113b9aae49e564c8a4c072c5c18bac077c52c304b","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=gpt-5.4 (xHigh)*; createdAt=2026-04-08T17:04:48Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":209,"at":"2026-03-10","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-10","status":"stale"},"measurement_id":"1b5b36cab771d6c2ccc7edabe53074ba795540a40730dd258b14524ab5cd5c45","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":"2026-03-10","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"cfa903819abdeebe393c5cac5f37e9e25b1d0c153e5cd7d29318ad5ecec2156f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.370000000000005,"score_display":"31.4","source_stated_rank":9},"run_fingerprint":"e08da6861ac6eac3483bacb5e6aba3044757939722738e229ff4e0a8a0f22308","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":209,"at":"2026-03-10","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-10","status":"stale"},"measurement_id":"12d151b87476c4b0ed1dfe434d8b8a68728257071626f3b46b714e7e1047d090","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_high","source_row_date":"2026-03-10","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"783fee3b7d191fa2773aa9b615c38b417e932b1df5870dd6cd1ac4d9bc4f5e7c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.490000000000002,"score_display":"25.5","source_stated_rank":13},"run_fingerprint":"33b613dcc7b9f83a66a05961191cda8fd6129d4c061322ee11ab78e7f65a043b","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"50bcf57912ac1019ce8584cbe75410c10d04d60af7ffd6c7558423d690503f02","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.329246,"date_is_proxy":true,"latency_seconds":209.424,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.04},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-2026-03-05"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"8d842f93d022cb42e1943441400092442464c14a2f90276f916d4890fabbdc8b","result":{"confidence_high":86.1794,"confidence_low":82.10260000000001,"sample_count":null,"score":84.141,"score_display":"84.1","source_stated_rank":41},"run_fingerprint":"8a482f9ffd06299d09430fccf057ca74780b8b9258749110781a2deca001f210","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d0b14a843de1594bf9587b67896aee034bc81b5c1ad132ad37e53ccc218921c7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 (xhigh)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"309df13fb2e5d9a3e71e2cc00cbbef6270e23dd5d77129313564b2d1f25d0f62","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.5972222222222,"score_display":"56.6","source_stated_rank":26},"run_fingerprint":"6a944825f691c7082b07bc731719850456298b233e94e26515b317797e974712","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6193c1b81b06e276afd006519b475b1973ddf5d1d5684650d45644785bd01ff1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.612174","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.4-high","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"a8155511d9bcff60b3d613babff808e217181f1c4acc9dc6eaad3cb20566ee0f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1607.0,"score_display":"1607.00","source_stated_rank":13},"run_fingerprint":"ab9bb5c54eaa845b935cf4ebd41d1524343454526bdf2a42443e9d883c2fcdfa","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9ebe1927c189c5764f1af966c8c764422b786b5dc00367e4657d8db389eb65b4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.26258","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.4-medium","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"dedb219e56b08d932bfc04826eaf8516384820f0dcf70f5d6b4ea16e078eab00","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1520.72,"score_display":"1520.72","source_stated_rank":18},"run_fingerprint":"eeabf21fdd055f94b46d39de39dcee8bbd1574e1c937f934021fe7d49f1e1fad","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"abb875d7bdcd5a3df49bb429e5ff654d6da4b60deebc1d284cb4213bcea3d8e1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.052314","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.4-none","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"d6bc2f99b9b22b2e91ed8409ad1c44e21807d97ae337d3bc36ad57110379a8dd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1086.03,"score_display":"1086.03","source_stated_rank":45},"run_fingerprint":"42fa4fe18478696e917e6ea113e70c9c4a1002e3660684ffac4fa7519e819f6b","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_da6182d6c493323255d5f78a","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cfb02d113a4d3fd85b677c16802a6b20923d68e1a6bfa8ec5bd42670d08ad33e","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":10.686976,"date_is_proxy":true,"latency_seconds":5174.848,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.843},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-2026-03-05"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"e52d72799fc61a03aa4c4d861b2b80c8f5a726edf86aadd4087aa88fcd7ffd35","result":{"confidence_high":76.91328,"confidence_low":57.928720000000006,"sample_count":null,"score":67.421,"score_display":"67.4","source_stated_rank":38},"run_fingerprint":"a06c5f0f853306ead493038ca530521bb8961571f22d8dafdd9b446a8f66d2e5","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"db2994267f546f914fc74417d1bfa89533487e78bc1f492b744c6e674b3cc196","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":7.50518,"date_is_proxy":true,"latency_seconds":1075.396,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.07},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":null,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4"},"run_count":null,"scaffold":"Codex","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"3cef72fe1831113803a418bd4b204479f0e6e7ab2a4dcffc0aa6b27c2494f537","result":{"confidence_high":58.411199999999994,"confidence_low":38.5368,"sample_count":null,"score":48.474,"score_display":"48.5","source_stated_rank":54},"run_fingerprint":"fec7a4cd7f30f11bddd532dec2a5ddc858e8f05f0c71ed70949462f58f22802e","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9c73f485b940f1b883066594","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"73bacf819d71b87f5f02370bc9ebdc499f6f5eb803dfab9527d9364061e5e853","metric_details":{"license":"Proprietary","variance":29.506060453832312},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.4","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f0bde9e6b26aa03e5f89a585922836263b4dd3cef17ff1e21eec5c6d6aa48dfe","result":{"confidence_high":1405.6966130277651,"confidence_low":1384.403767638357,"sample_count":3801,"score":1395.050190333061,"score_display":"1395","source_stated_rank":89},"run_fingerprint":"b5fa0c6947df58c130cefb55f31d53cff6497fcb0df1c76f0898a89b4e1cd60a","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e5f556a0e36183d119678e30","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a727a01b5cca49455b212322c2dd18eaaa1125323d88489d5725b74de28d907f","metric_details":{"license":"Proprietary","variance":92.76471835862908},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.4-high (codex-harness)","subset":"webdev"},"run_count":null,"scaffold":"Codex harness","tools_allowed":null},"protocol_fingerprint":"f9e0bf808d3ecafa142386e5cb7f90324af0a073350b0baa5858e7ea268b058e","result":{"confidence_high":1483.8192491148734,"confidence_low":1446.064681413046,"sample_count":1351,"score":1464.9419652639597,"score_display":"1465","source_stated_rank":63},"run_fingerprint":"663568a90e09b7afa708f3834dc0e1f73dd38e0330f350f15ea91098432ff0cf","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8d339d3da0dba97fb8ee52a5bb653bc4ad65842b757b172d3bae03d9096711f2","metric_details":{"license":"Proprietary","variance":87.09517861157653},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.4-medium (codex-harness)","subset":"webdev"},"run_count":null,"scaffold":"Codex harness","tools_allowed":null},"protocol_fingerprint":"a1d96efdefd27ed7cbd808241dfd08c48a27bbc99c23b3c87146b36f0ae963c3","result":{"confidence_high":1462.172385421994,"confidence_low":1425.5897369603329,"sample_count":1376,"score":1443.8810611911636,"score_display":"1444","source_stated_rank":69},"run_fingerprint":"521858fde1f50eab9be79efc0019a0c82c3dcebfb562a37bb66e802bd001301b","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ddf778813878f411c6dcaace8a2acb4b3a8ffd3a46c5501fa4e1300e8d9a231f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"b24adae8611ff00ecf999c3b605c87c5d180fd28a299daa6f6e8afdefdc66ee9","result":{"confidence_high":1482.43,"confidence_low":1443.55,"sample_count":1277,"score":1462.99,"score_display":"1462.99","source_stated_rank":64},"run_fingerprint":"5a934d28d2b273b6bc97288439b0f2786d467ccfdf0fca03fc907b4bc6d06581","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"88ad2012717457cfc7dcb84d49b669ce3ab581516086dcb04d53e70288db9388","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_medium","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"da747f351bec6022827385a66a8015297c000abc6a2a09a59ef289b6c932fc8f","result":{"confidence_high":1461.33,"confidence_low":1423.97,"sample_count":1316,"score":1442.65,"score_display":"1442.65","source_stated_rank":70},"run_fingerprint":"a6420bc8a2eec4d6d14466494bf24db9e3c8ffb307eb2e91eced9f33f5b878a6","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4172fd9d73942fb0a8346db26871c428140ad1ded6c6ec334dec87fbcdcbcebe","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"1ffad1a92cd4d00e398bf2f6b7dbc9ae55783f564b5f18d3cac63a969d09bfc4","result":{"confidence_high":1399.53,"confidence_low":1373.94,"sample_count":2493,"score":1386.74,"score_display":"1386.74","source_stated_rank":91},"run_fingerprint":"094e3139b720c52e6f80cb8efd34dfd17b6b7cde87bbae287d135c74c0e20f14","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4f6ed1eded242eef5666c35e","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"mirrorcode","metric":"mean_score","name":"MirrorCode","release_id":"release_07bf24088c0805f7b6df7a88","split_or_window":"15-programs-30-language-tasks","unit":"percent","version":"ML +Private 2L"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T16:12:45.299Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T16:12:45.299Z","first_observed_at":"2026-08-28T19:57:34Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"38816be46342d52a83bad2a6f6a5b6c943134131e70a195f1d00786b3ddbdfef","metric_details":{"best_score_across_scorers":"0.15555555555555556","model_release_date":"2026-03-05","stderr":7.7094504410037406},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mirrorcode-ml-2l-gpt-5-4","log_viewer":"","logs":""},"run_count":3,"scaffold":"Epoch AI Inspect ReAct agent","tools_allowed":"shell, text editor and visible-test evaluator; no internet"},"protocol_fingerprint":"a501a42ac912129da6c4f1f0956794f1706aaccd3f4734f82fa0024d9ba56b28","result":{"confidence_high":30.666078419922886,"confidence_low":0.4450326911882243,"sample_count":30,"score":15.555555555555555,"score_display":"15.6","source_stated_rank":7},"run_fingerprint":"1483c786be3986e166c938929bbe1942fd71c08c929688e930fa491d5d96b6ec","source":{"content_hash":"3e212e97e03d0343e676ecc7ff046ee7b94d219ba7d863364634b7361dc89b4b","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://epoch.ai/benchmarks/mirrorcode","id":"epoch-mirrorcode","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · MirrorCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/mirrorcode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3a067d20f8fb2cfdce407f0bba04eb25c993c1bee783904702832fd7c869d29e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"a2419b920174c9dae8ef843b6ee83c439c8799f276a1531e3f5f834e17649ceb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.7,"score_display":"77.7","source_stated_rank":23},"run_fingerprint":"08f8b0acf2c915843ec1d0561848d5650e23372ca63b404425979b9b1b225473","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8cc9385efbff116dfd7d689f8e08532dcbe8664553289434b66a92f67bb7c78c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"8eb8f6ad04664326eab658717e72661bbd4aeea0e061088b7b54bb1461d19426","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.440000000000005,"score_display":"57.4","source_stated_rank":54},"run_fingerprint":"b8531c48010afa080608c0042f72ea635ae0dd1cfa22ccac81c4a76249bb0e0e","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_da6182d6c493323255d5f78a","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":207,"at":"2026-03-12","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-12","status":"stale"},"measurement_id":"a8c1c87237d10204dd4b96c6c9e4de1f7cb80f5bf3b02b96a7224531599b63d5","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"standard_error_points":2.0,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.4","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_unknown","source_row_date":"2026-03-12","source_scaffold":"ForgeCode","upstream_leaderboard_url":null},"run_count":null,"scaffold":"ForgeCode","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"3bb4dff67e06522dbbdf15384e24722bcea9e17a4ec12d130883e2fff68d8db8","result":{"confidence_high":85.72,"confidence_low":77.88,"sample_count":89,"score":81.8,"score_display":"81.8","source_stated_rank":5},"run_fingerprint":"7115bb50ad21ac004d3fa05a911572fccb97b7efd6ef824296d79249f68349a5","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_287dbaa4fb64c548fbd68765","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":152,"at":"2026-05-06","basis":"evaluation_at","evaluated_at":"2026-05-06","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"7c8ba78c7015978111a980d5c1636de7cddfe87d9eb6fd711c88b0621e792f03","metric_details":{"cost":1.8423,"pass_2":29.55,"pass_3":24.48,"pass_4":21.65},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"gpt-5-4_sierra_2026-03-25","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"1f2c749156758dd2cdd932f2c0fcb8e343f009eac9e98fe705311767aaa43ca8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.43,"score_display":"39.4","source_stated_rank":null},"run_fingerprint":"1653a01bc4203acfe9c161525fc0ea84a59c583a2a6198b3d5a19eeb78550d12","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":308,"at":"2025-11-30T18:41:10Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-30T18:41:10Z","status":"stale"},"measurement_id":"c15d2c5c0285c08d5d47224e1e4a3f1a9aa9128105c6ae73cefd439debc74fa4","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.8,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gpt-5.4 (xhigh); createdAt=2025-11-30T18:41:10Z","source_row_created_at":"2025-11-30T18:41:10Z","task_pass_coverage_threshold_percent":75},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"gpt-5.4 (xhigh)","source_reasoning_label":"xhigh","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"363965a09a1cc4d9b8ee0fb46d2b71b8bec2fca8de037ca2b8690a3b995d63bf","result":{"confidence_high":73.39999999999999,"confidence_low":67.8,"sample_count":1000,"score":70.6,"score_display":"70.6","source_stated_rank":13},"run_fingerprint":"e4a433a08311921dc1e1f084852d0e663d6d87448d4b8076d38752500fcbed83","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=gpt-5.4 (xhigh); createdAt=2025-11-30T18:41:10Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"5a7d89cf439a9337a50a24b1045149a8fdceca2895b02e4ac130feb7d453f741","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.527","mean_standard_error":"3.4","model_release_date":"2026-03-05","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.8,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5.4 (xHigh)","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"84587636e0c088be4a97d1e7f7919c4b205a7707c511b92d92ed817e3e031d46","result":{"confidence_high":43.448,"confidence_low":28.552,"sample_count":480,"score":36.0,"score_display":"36.0","source_stated_rank":16},"run_fingerprint":"9dd13e0d9db72a23a6546e6acbb16d1aa1cb394e1ae6339d10ad5201acea1c7a","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0cb8e1a21336ec26907fe90e3be8062e90f0c30228ea954132b1d7840da045d4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.682","mean_standard_error":"4.4","model_release_date":"2026-03-05","notes":null,"standard_error_points":530.0,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"a458b4d36083d2aca53818373f0066e5717458cacfecca91130c83525af3ffd7","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":52.400000000000006,"score_display":"52.4","source_stated_rank":21},"run_fingerprint":"a64464977c1b2e5f3122d4eb375dc93d16b0d3a64b47d5ccec67b791d3338d21","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dcc0ba793d70c2baa40389f5","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"da36231e31182f23e5ed7888af77b5d822d7edc57ae4a2d745d5a3c7f6bc152e","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":81597},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 5.4 (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"535b117b52dbc88aca21b4ac1248ee3ed0f0a3b0db22fd256f25306d82e6e395","result":{"confidence_high":0.01975041291236446,"confidence_low":0.0013255801637096012,"sample_count":4332904,"score":0.01053799653803703,"score_display":"0.0105","source_stated_rank":28},"run_fingerprint":"91b314b9df67875cddf22e95e902545abf023ba8181aa0e2c93f445c346d1280","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4e148d1e9dfcbfb1ad710c65","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a95f0cf5da7b01034a5125cda40e5dd20278bb8b88e52ac1915f2961a7015f82","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4","source_effort":null,"source_model_version":"gpt-5.4-2026-03-05_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"262915de42a64a04effa6b5004cbc4c44985ca1f5c71ead94e759a23fff7f15a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":6144.176,"score_display":"6144.18","source_stated_rank":17},"run_fingerprint":"17981075285d73f90299a1e64667b1afb0e0a7dc12f077ea0548723a5bf68cb7","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cf26814a19699b9dfdf18c84","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fcef5209df2c4872cdb4a6ae75992cffae111571e24c492e47425b4c4288c395","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1050000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-03-05","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · GPT-5.4 (Xhigh) · gdpval"},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-4","upstream_model_label":"GPT-5.4 (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"d09302ed87086e7bd7a7c586f2261cfc6b741a54e1a2eb9a1f0b3568e0363a18","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1232.67,"score_display":"1233","source_stated_rank":null},"run_fingerprint":"004dab57db3cf8d76119a3a9afc051148122c8c3890625d7d6bb12824fc64b54","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.4 (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d50b48d5936735014a249939","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifbench","metric":"inst_level_strict_acc","name":"IFBench","release_id":"release_4adf348ac2f76539ed7ccb10","split_or_window":"ifbench-test-300-prompts-344-instructions","unit":"percent","version":"swallow-evaluation-instruct"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-04T14:29:18Z","observed_at":"2026-10-04T14:29:18Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6478a2d63e05d38ba4833bc40b8a15c9e9626e080c7d248bf6c0c1ac7a0c4a68","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","instruction_count":344,"model_release_date":"2026-03-05","prompt_count":300},"model":{"id":"openai/gpt-5-4","name":"GPT 5.4","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"rule-based instruction checkers (lighteval 0.11.0 with Swallow fixes)","run_config":{"decoding":"temperature 0.6, top-p 0.95","leaderboard_revision":"c58a13a852eba061d639af7caf4ddbdff33b36e3","protocol":"Swallow LLM Leaderboard: swallow-evaluation-instruct (lighteval), allenai/IFBench_test, instruction-level strict accuracy; the final answer without the thinking part","swallow_model_id":"gpt-5.4-2026-03-05","swallow_reasoning":"on (medium)"},"run_count":null,"scaffold":"swallow-evaluation-instruct (lighteval), swallow|ifbench_singleturn","tools_allowed":"none"},"protocol_fingerprint":"c558a93bada10f3c0600544169f1ced776f37708edc99457d40439adbfefbf2d","result":{"confidence_high":null,"confidence_low":null,"sample_count":300,"score":65.10000000000001,"score_display":"65.1","source_stated_rank":null},"run_fingerprint":"966dac17bdcc7a606f261481d43ffb77a5aa92151df9604c53f3da35e81463a8","source":{"content_hash":"fffdc5bb4b3de214a8a4ce885cf286e13a4b1488d9a4a0791e5918d1918b74cb","fetched_at":"2026-10-04T14:29:18Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://swallow-llm.github.io/leaderboard/index-post.en.html","id":"swallow-llm-leaderboard","last_fetched_at":"2026-10-04T14:29:18Z","license":"Leaderboard content CC-BY-4.0 (Swallow LLM Team, README of swallow-llm/leaderboard)","locator":null,"name":"Swallow LLM Leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/swallow-llm/leaderboard/c58a13a852eba061d639af7caf4ddbdff33b36e3/swallow_leaderboard_post_en.js"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e80950dca1866f561008c61e","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"93065259c8cacdfb1224d4df88041a08f6c107eeb3ded001e6a2e5ceffb9ccab","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-09-02"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Muse Spark 1.3; model release: 2026-09-02","source_accessibility":"API access","source_model_name":"Muse Spark 1.3","source_model_release_date":"2026-09-02","source_model_versions":["muse-spark-1.3_max","muse-spark-1.3_unknown","muse-spark-1.3_xhigh"],"source_reasoning_efforts":["xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"175ad500b325fe73f1e372e566382e62241a1b943e2a7b2c88ea0425b3fd8101","result":{"confidence_high":159.56,"confidence_low":154.73,"sample_count":null,"score":156.86,"score_display":"156.86","source_stated_rank":17},"run_fingerprint":"24d7436819a95344e586432688eb5990cd4c003a1dafe95c61a2b915fdbda8d5","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0af9228b335da865d7243f2e","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"456e8d8f3de92cc5b6d1b63f56f5672dff303d69eab5000ca68d4b1e806266a5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Max) · intelligenceIndex"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3","upstream_model_label":"Muse Spark 1.3 (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"e2daf4267eee6d155bfa13e6e2d13e1391f7c0abe354e335f2238964d8c704bc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.092311,"score_display":"48.1","source_stated_rank":null},"run_fingerprint":"ac0f4bb743178a4681f826afd57a1c31c48ef5d0250ffd05f7423823883ceda3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a422b83f1ec7533b2f159f5a7dc3d687f1b8616fc7aefd1c1b66e4c7526dba4b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · intelligenceIndex"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3-xhigh","upstream_model_label":"Muse Spark 1.3 (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"86ab4159ceded0db6d950839d7baf9ade26a600d0b99d96da44cc7c5033449f6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.073297,"score_display":"45.1","source_stated_rank":null},"run_fingerprint":"3ff144099e13851a79c0f8ceab6855ae06969fced024740b466569beb246b699","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e1c1488c0ba16485203448d4dad5e2c20265fc0307f7009b5026b0d2e08cca7d","metric_details":{"license":"Proprietary","variance":9.924231895011472},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1500.4780227118652,"confidence_low":1488.1291720525194,"sample_count":12343,"score":1494.3035973821923,"score_display":"1494","source_stated_rank":9},"run_fingerprint":"52b02dc3b37a44049c5da4ef8a923e1321030a9c46db23441c3f789323ea9eaa","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T23:43:32Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e0be7c590398905ecfcb5a295c5f13d34e9c69bc273a6a6ed8add460f678c811","metric_details":{"category_scores":{"Agentic Coding":64.091,"Coding":81.0625,"Data Analysis":79.56533333333333,"IF":78.00425000000001,"Language":82.79366666666667,"Mathematics":95.94999999999999,"Reasoning":89.65375}},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.58864285714286,"score_display":"81.59","source_stated_rank":7},"run_fingerprint":"cddc486f5781776cabb5f926d1446db91b557917357c72ee96877dff45756f98","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"01e7d850e606ef558f0e38e0bbb8d0152b79da4405915f85ec4259a9457d0dd4","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Max) · hle"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3","upstream_model_label":"Muse Spark 1.3 (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6d4a0533e6f2eb8eb1b74ae0efb95f35de675209fa8355ae6beb4b613fc7fe81","result":{"confidence_high":50.811825,"confidence_low":46.59779,"sample_count":2158,"score":48.702502,"score_display":"48.7","source_stated_rank":null},"run_fingerprint":"bcaa08f31f0d78bce5882b23f127b9b24ad8e44d61afe7a16ebf95abd05d52ce","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4ffab8e7589762be494755b73c91e94c6fed653724891ec326fec8223ea70870","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · hle"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3-xhigh","upstream_model_label":"Muse Spark 1.3 (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4cf0c1ce8199624ba82cd2317e043f25f0af004365b22bd91d8d2a2d9ba71cc9","result":{"confidence_high":49.560864,"confidence_low":45.350882,"sample_count":2158,"score":47.451344,"score_display":"47.5","source_stated_rank":null},"run_fingerprint":"c97a9296c0de36d90b2c6494503ef34971297c797990ffad1468b9d54b0a2cf7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6e5480990705a3bf0c80758bf06ee2f09789e35759375e1c99aa6c903ecb4d48","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Max) · gpqa"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3","upstream_model_label":"Muse Spark 1.3 (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"23532e1ab096039f6b9389b68faf9f4ea4f2a2e25a9043ce236dde20b7d59262","result":{"confidence_high":96.198914,"confidence_low":89.2146,"sample_count":198,"score":93.535354,"score_display":"93.5","source_stated_rank":null},"run_fingerprint":"ac153f9f39d52370ac34e9fcfd0ce65344a48b814d6349ca2ae06d220998a24c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Max) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"17414119a0b06c1bc308e6d92867f36294c7e40c4f774cb5bb3dbed217a3da13","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · gpqa"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3-xhigh","upstream_model_label":"Muse Spark 1.3 (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ff4592403a8154313aaf1b32b5855f78360053c133fc6484d82a3eeadd225844","result":{"confidence_high":96.648379,"confidence_low":89.954185,"sample_count":198,"score":94.141414,"score_display":"94.1","source_stated_rank":null},"run_fingerprint":"381917284abbbad41fc3a60b37a52ee9990dae1a16181d49bcefd57f2320114e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":18,"at":"2026-09-17T13:30:27.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-17T13:30:27.000Z","first_observed_at":"2026-09-18T22:27:39Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4145f13e61bce60a74266ca9af25684e3154b21d1847a09a3d94efc1cecc7248","metric_details":{"best_score_across_scorers":"0.9916666666666667","model_release_date":"2026-09-02","stderr":0.6154574548966636},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Wmxkx5m3qaQuwzem6gzZmg","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"33ceb2e0d810bdbafc77f518cba2ee9be1b6c409c51b20f9b5f61314928fff38","result":{"confidence_high":100.0,"confidence_low":97.9603700550692,"sample_count":45,"score":99.16666666666667,"score_display":"99.2","source_stated_rank":15},"run_fingerprint":"c3e55d1caa980b246c7db9f1558a29f3d6b5293fe15bbe2e2d3d514f793bdaf6","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":16,"at":"2026-09-18T16:34:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-18T16:34:54.000Z","first_observed_at":"2026-09-21T14:13:55Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"717c9890c58e1fce1dd8d40e4bfb88bd923c2dedb7f29eb6e7ff525cbdd8a1fe","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.7403508771929824","leaderboard_private_problem_count":285,"model_release_date":"2026-09-02","public_example_count":10,"stderr":2.6016750822385526},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"6735DShGR4Zu5KZ2U6n4pC","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Muse Spark 1.3 (max)","thinking":null,"upstream_model_id":"muse-spark-1.3_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"86c9feb2114dca3e6138829322e72fa076845a3cc3f734638a7c5460af8e3d54","result":{"confidence_high":79.1343708804858,"confidence_low":68.93580455811069,"sample_count":285,"score":74.03508771929825,"score_display":"74.0","source_stated_rank":20},"run_fingerprint":"5272a7da579a5889bda16be1559120fb955fba7ff1a6a0aa438a22a5f977e063","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":18,"at":"2026-09-16T22:00:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-16T22:00:25.000Z","first_observed_at":"2026-09-18T22:27:40Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"293707e98aff71127dec6cc5eda6efe1951d89af91bef1fc467b1850d2926dd3","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.743859649122807","leaderboard_private_problem_count":285,"model_release_date":"2026-09-02","public_example_count":10,"stderr":2.590152474777329},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"JeYksd3f2RkegmxCcZyBMQ","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Muse Spark 1.3 (xhigh)","thinking":null,"upstream_model_id":"muse-spark-1.3_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"2e858b97a31709ccb4fb9ac0e4efa45a6ace1a762e1b6beb5970864935787323","result":{"confidence_high":79.46266376284426,"confidence_low":69.30926606171714,"sample_count":285,"score":74.3859649122807,"score_display":"74.4","source_stated_rank":19},"run_fingerprint":"4785d6cf5d12d640a5dbdfbb474a97b1152698f07658927444003de3f0a00603","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":16,"at":"2026-09-18T16:34:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-18T16:34:54.000Z","first_observed_at":"2026-09-21T14:13:55Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f6364652b2499ddc8b1d5e1a790053aac1ade488cdab08dc67ec298861475c15","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.4634146341463415","leaderboard_private_problem_count":41,"model_release_date":"2026-09-02","public_example_count":2,"stderr":7.884502378168105},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"naZno5TPpgMJfv2b5mhvUW","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Muse Spark 1.3 (max)","thinking":null,"upstream_model_id":"muse-spark-1.3_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"e0c81ff631e2766c70cd005a9495ad590823b345eb9f683bc4bd3141d697ea7d","result":{"confidence_high":61.79508807584364,"confidence_low":30.88783875342466,"sample_count":41,"score":46.34146341463415,"score_display":"46.3","source_stated_rank":23},"run_fingerprint":"b98de9228ca73adc2b2dbc707d0790e41e5851cc667dea9aec7e210e510f1cf3","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":19,"at":"2026-09-16T02:51:59.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-16T02:51:59.000Z","first_observed_at":"2026-09-18T22:27:40Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4de2783aa7729acbd2dabea45d50429b893533137acf018baf225795ceff9d9c","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.4146341463414634","leaderboard_private_problem_count":41,"model_release_date":"2026-09-02","public_example_count":2,"stderr":7.789619230571372},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"UdTt275jahRApXURferb9i","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Muse Spark 1.3 (xhigh)","thinking":null,"upstream_model_id":"muse-spark-1.3_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"60472623ee71562a378a1fcd7e0cec76b19334304d5405eb20c6f9df10a655f6","result":{"confidence_high":56.73106832606623,"confidence_low":26.19576094222645,"sample_count":41,"score":41.46341463414634,"score_display":"41.5","source_stated_rank":26},"run_fingerprint":"62ccb3921de63b0097968289fffbf0d2c393b64462c1ce8d3904d1a2a1a928b6","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9018092a67f40b82b02761318523b8528b421a9c21a84dd587fe35a62dce8b46","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · mmmuPro"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3-xhigh","upstream_model_label":"Muse Spark 1.3 (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9032869fcd371c939bff9effbf31db43fc61d0f03cc14e2c7fc20df823085103","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.023121,"score_display":"82.0","source_stated_rank":null},"run_fingerprint":"09dd522bbbf6af91379f1646ce6dee7e1bf47ec071b77222952fc90692a64547","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9c1b88e7a0269413c2bbbd96cf86583afeec1e6861eceb0a2e86bab4bc8875cf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":null},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3 (max)","source_effort":null,"source_model_version":"muse-spark-1.3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c171c621ab5538175253917c510e3b17320ee80de5c3c32cde9c0d842dc20fb1","result":{"confidence_high":36.024012,"confidence_low":16.271426,"sample_count":71,"score":24.8571428571429,"score_display":"24.9","source_stated_rank":39},"run_fingerprint":"1489fe875980b666da3d2c0e393411630c698219c71eceb772f76d41281db027","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"51b676a384c7f8a2387881d49a81e48ffea809b6e3fdf484273819ef15a77882","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Max) · critpt"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3","upstream_model_label":"Muse Spark 1.3 (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6072aad4d58cb947ebea47bc80aed384484d4fa03f8eef8a0364f0ecdeae9c3c","result":{"confidence_high":36.10929,"confidence_low":16.221104,"sample_count":70,"score":24.857143,"score_display":"24.9","source_stated_rank":null},"run_fingerprint":"e7807930e529179f132800e250b1f8b4b37a03bced7b085816304003dde14aec","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"05bd20c0a219d2e788f88a49a55fbd9876fb4f4cef95a95aa4ebc5e1628dfa13","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · critpt"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3-xhigh","upstream_model_label":"Muse Spark 1.3 (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"acd202e0c402e6d10f933ecb1a64cfafb0fb5b89cf2eb0fd29bc7acf14dc99d7","result":{"confidence_high":37.331,"confidence_low":17.166194,"sample_count":70,"score":26.0,"score_display":"26.0","source_stated_rank":null},"run_fingerprint":"54a2d5768622f17f7325b686a80495d543e4bcd9ca93c39e6f6384d6b46409c6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":16,"at":"2026-09-18T19:12:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-18T19:12:25.000Z","first_observed_at":"2026-09-21T14:14:00Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"26dac6265f7e58cf5191bb83583cb82d0c4974bc2a87efa087bfb38020ccc17a","metric_details":{"best_score_across_scorers":"0.38","model_release_date":"2026-09-02","stderr":4.878317312145634},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"iPdvXwCfjkWX8AcpsjR4sZ","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"bbad421cef15f4cb6def126a94377a1bd6e759e8e574c5966378ca782b701aeb","result":{"confidence_high":47.56150193180544,"confidence_low":28.43849806819456,"sample_count":100,"score":38.0,"score_display":"38.0","source_stated_rank":32},"run_fingerprint":"4d081a959f04bf33ec32131dc5a41eff4696c79c709c0a9501ce330b66d40725","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":19,"at":"2026-09-16T13:39:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-16T13:39:25.000Z","first_observed_at":"2026-09-18T22:27:44Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0c9e3c3a91b691691be596a4ff8ecfd8683835898fe90d95ad1b9739615c7703","metric_details":{"best_score_across_scorers":"0.35","model_release_date":"2026-09-02","stderr":4.793724854411022},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LGWtZA9WGoRRnebM4Q7VCV","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7f356895dc9b0b7e526a4cc6a493cbb60529f07f30e45ba634b81c23aa535be0","result":{"confidence_high":44.3957007146456,"confidence_low":25.604299285354397,"sample_count":100,"score":35.0,"score_display":"35.0","source_stated_rank":40},"run_fingerprint":"ce531cdb13eaf6989bccc5feb19be5b53944136d3c47310b740f64a7c864abe3","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b8180d07b35a8b28c91071a55c62673f6c065267a22d200580925517914a51bd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.64","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"steps_per_task":"98","tokens_per_task":"52005","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3 Max","source_effort":"Max","source_model_version":"muse-spark-1.3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"3fdf6f5c37f0d5a1d8aee488c6af770cc887b7df53df11c4f76231ef40474998","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.6,"score_display":"41.6","source_stated_rank":21},"run_fingerprint":"8f15e3d45d072db7a77a6bdccd43f14ae2b395ea45b4d708f6725753e8a695b3","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b9df60d89524e66ee31864a4764a798dca80a8eef0a6f6fbde7924ba88e2f322","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.1","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"steps_per_task":"83","tokens_per_task":"40891","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3 Extra High","source_effort":"Extra High","source_model_version":"muse-spark-1.3_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"8dc78847a7f3280b3504cd8dcd09b70c461d2078fb503ad07aa0e8c9f342135b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.5,"score_display":"37.5","source_stated_rank":30},"run_fingerprint":"263b33c1c14beee739a2573576e08aec81baf21688f7e527fb7dca22f9cc9015","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2de59c97dac130a87e15dc8a95297b05bc6991d82d20d65e18274328b64a81b7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.66","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"steps_per_task":"69","tokens_per_task":"30654","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3 High","source_effort":"High","source_model_version":"muse-spark-1.3_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"ad25ba050dc6d43d5fa1e729fc5dad63d894bd31e862fb0204202fa96869d77b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.4,"score_display":"33.4","source_stated_rank":38},"run_fingerprint":"af5019c4c53cd23883957ffd338b02fc8ea350025fbb48498de6bd07f29c9a8d","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_79d4ca0dd775ade9d18690ed","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"092b11d38f200ff043241140ae458429cf193fa2d9dc9295aaa3741d16092415","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.49","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"steps_per_task":"64","tokens_per_task":"27255","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3 Medium","source_effort":"Medium","source_model_version":"muse-spark-1.3_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"3c1c5f93d8f7eddced2dcbc7f4ada0c0a83a37044300c3ffe3b2f5a136dd397f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.6,"score_display":"32.6","source_stated_rank":42},"run_fingerprint":"97fcd69ede4f084d1b3d6612ca0777245721eb1368e93eb174813abcc16b2fad","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_37df727571e09f570c0bfd81","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"125af8b7c8fb8f4a3f713ae5f39c316cf9784640e8e1d6690702f4fd55efb27b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.93","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"steps_per_task":"47","tokens_per_task":"17483","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3 Low","source_effort":"Low","source_model_version":"muse-spark-1.3_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"62e2b520b0fd7b8ec2bc0893e920fbb077a48df32979ea969a88703ea675e6bf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.300000000000004,"score_display":"29.3","source_stated_rank":48},"run_fingerprint":"25c22ee3c46634763f860ca71c9d49f6287127dea7f77e33205ee365ab10bc2a","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_5cc70bd262d5782faab955fe","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ab0df8fbbf53d6562196ce4a87a8e1f7ee1a1b217e54e10423cf54fc8c2927d2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.56","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"steps_per_task":"34","tokens_per_task":"10620","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3 Minimal","source_effort":"","source_model_version":"muse-spark-1.3_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"dac1948af9c4efe751f1a6cbbb8f519ac8bda2060ad721ae9fcf2818723df3ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.3,"score_display":"24.3","source_stated_rank":54},"run_fingerprint":"6513386730c709fc18ad7ff1b6202969cf0a31c01fda0faac00698438d098393","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_ebb5ae2b5527f831ef1b7cb0","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"33798a7541978cd0bfd17ac3e2f0d96a1bb1961f5ed2fbed69357224f092a2a5","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.360741,"date_is_proxy":true,"latency_seconds":1791.133,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.448},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":131072,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"meta/muse_spark_1_3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"90c325dfcc4528d0bdbb19da554ff1697e8bd6574389ac7966bc88edfdfc5e33","result":{"confidence_high":46.78208,"confidence_low":41.105920000000005,"sample_count":null,"score":43.944,"score_display":"43.9","source_stated_rank":29},"run_fingerprint":"0eee6866d1e68abbd25bf8c7c3969923151a5a2e8b8d7b9f5b517f235e1168dc","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8b4e535e7eadc22528e1a2a3116e176d4ad1229fdda0e3855e097e100b2f713a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3 (max)","source_effort":null,"source_model_version":"muse-spark-1.3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"512161504722602312d09375201dda21a04e4c214c82fc65885eafb6d38093a2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.796296296296305,"score_display":"58.8","source_stated_rank":15},"run_fingerprint":"99e85714dc4d53a847ceb65ffce464c8f0a257c273e70fdf49d1943cf8d160b0","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b0418855aaf8e2996754b6d9e5ba3add9cbadb6a50a21102cffeead52ae8c581","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3 (xhigh)","source_effort":null,"source_model_version":"muse-spark-1.3_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fdedc3baa28b0a658d8497c404b8130e289fdb5c222c1ca6ab081aef6ad16f3c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.7222222222222,"score_display":"59.7","source_stated_rank":9},"run_fingerprint":"d239e03ccbdea4812c01ed96a154a51ae475cbd5bdf63584d6ebcd0d9576bac3","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6161d129cb028d323e9419aa285c66b14173e6ccb980de91a7791fe7d14a7b7c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Max) · scicode"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3","upstream_model_label":"Muse Spark 1.3 (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1be611f83a2b53c6bdcbee487de86e4571e0bb8b17337f35bdb4f84d79552c56","result":{"confidence_high":64.328796,"confidence_low":53.032219,"sample_count":288,"score":58.796296,"score_display":"58.8","source_stated_rank":null},"run_fingerprint":"336d6c0b0c55db810362f7a5923ce8249fd311990c1b250321f81b2c22585bd3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3911f712d3bc5a938b1964e4e4269bc38015946ebcad9908d745abc2e713efc8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · scicode"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3-xhigh","upstream_model_label":"Muse Spark 1.3 (Xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"69cb9b3229e95ca1383659844fee11dc481bd753a3fab2afcdb7dfd9cd96355c","result":{"confidence_high":65.222782,"confidence_low":53.965709,"sample_count":288,"score":59.722222,"score_display":"59.7","source_stated_rank":null},"run_fingerprint":"3e73b3596eaf4fc6a8b0039e7e1e5c70a50491816770c57a8b6126a96fb19dd5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"819f34ec1827c1a7033ecbf2527119b972a9cb3dea6562589e046260f8938c8b","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.101762,"date_is_proxy":true,"latency_seconds":773.797,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.901},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":131072,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"meta/muse_spark_1_3"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"d82de6437b226c835196f0cea5753caa6daf658cfc5e0ea75299685433bfe8b4","result":{"confidence_high":88.54495999999999,"confidence_low":77.17304,"sample_count":null,"score":82.859,"score_display":"82.9","source_stated_rank":16},"run_fingerprint":"6708928974aa26e3c5796e67ee69ef8787e2c7519d69f1d76066bb083563b39f","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1f9af3e38126676903aa2e80f8f2c4245c2fa92601a6cce4716c9910aeb22c83","metric_details":{"license":"Proprietary","variance":19.022366961944883},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"muse-spark-1.3-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2d5de052eeaa838985afd4f539c2a56b97bfab270d76a7976036de9f75b92d01","result":{"confidence_high":1665.1209031687533,"confidence_low":1648.0242790158459,"sample_count":7249,"score":1656.5725910922997,"score_display":"1657","source_stated_rank":14},"run_fingerprint":"9d29a64df880cda490e682505bf8e083fd8e4e191194ac43c97ae3d8d1aa1bc0","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4ec0827d63e225b328082e43dd4490aa8413020de2ed21ac3c1cc81f4dd68016","metric_details":{"license":"Proprietary","variance":19.456636570122914},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"muse-spark-1.3 (xHigh)","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"34273f1c801e883dbabcbdc77c2824d8ab9e80a6cc9713494d7c4a9f0dc85157","result":{"confidence_high":1632.8461324529349,"confidence_low":1615.555456564082,"sample_count":6326,"score":1624.2007945085086,"score_display":"1624","source_stated_rank":19},"run_fingerprint":"e499253f2b5b68095bbd7ca862336e023dcd9dd1ca0fa6fd9d8bf282f2920802","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fc756f90b612680878eecec05545509c1780972dd115865887855a619968ec14","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"muse-spark-1.3_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"495390f6068b33f67499742dd1f557fa37fd788e81acc14138326d1f52d8e992","result":{"confidence_high":1664.07,"confidence_low":1639.8,"sample_count":2972,"score":1651.94,"score_display":"1651.94","source_stated_rank":12},"run_fingerprint":"adb26ed0f724d70346c63223c6640cf67436f9beccdbc582cd58bf156f224236","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"981c917566d63a10afdd364ec302d71deebfd09e0babe66417f3374dda5f9a2b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-02","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"muse-spark-1.3_xhigh","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"fd179cbc33b200af5936e86ed93600ba4267d5c282b51708d5d03f2076da7d29","result":{"confidence_high":1637.13,"confidence_low":1609.11,"sample_count":2123,"score":1623.12,"score_display":"1623.12","source_stated_rank":15},"run_fingerprint":"d178720f6c30c5b57b9c7e3bf2da708ae78c30fd4c3d8ab88b4dab58b8892747","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ccfc0808de44cc3a4acf2925b6c1fc216529ba472c603c339f37779c20744f60","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.663249,"date_is_proxy":true,"latency_seconds":857.636,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.375},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":131072,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"meta/muse_spark_1_3"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"11b6c9a166561931c50573f5540a16df3f20d649b2021b73ab73307d118f62a6","result":{"confidence_high":73.02,"confidence_low":71.55,"sample_count":null,"score":72.285,"score_display":"72.3","source_stated_rank":24},"run_fingerprint":"d1d44c0ce6dcb54e58dfea1c7622e21db103773857813a70e68f16ce3444c452","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a719e631cd162911e301ff96ef79dd516d52bb2f8f247559b2242051b7aaaebe","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Muse Spark 1.3 (max) · terminalbenchV21"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"muse-spark-1-3","upstream_model_label":"Muse Spark 1.3 (max)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"1c2a9d4ebed119370847fdf59803fc745eb73032d3d9fa9e8024d6d9add89d65","result":{"confidence_high":90.392278,"confidence_low":75.311028,"sample_count":89,"score":84.269663,"score_display":"84.3","source_stated_rank":null},"run_fingerprint":"afe728da267ff8da78f7fdbacd66aeb24680972b9a79821eb2e6d0f370973f12","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (max) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":32,"at":"2026-09-03T09:21:03Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-03T09:21:03Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a2cc37bf12cd8c2bf057a790d9f4231c4608393865dfe384468abcf1f4dbe95b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Muse Spark 1.3 (xhigh) · terminalbenchV21"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"muse-spark-1-3-xhigh","upstream_model_label":"Muse Spark 1.3 (xhigh)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"35ccda1d5fb3cd8c4459ef7dbe32ba65c4cc8b2eb6cd337c976588d6b0c29a5a","result":{"confidence_high":91.260622,"confidence_low":76.596889,"sample_count":89,"score":85.393258,"score_display":"85.4","source_stated_rank":null},"run_fingerprint":"bf9b074cee0b6c397bf61a1c60ce7e2de3d1405dd6ad17ce4a02fc312667f085","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (xhigh) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"24d3bc6be3048022b113c0d183d6fa44074685d09c43d504f12ceaa9e3cd9caa","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Max) · tauBanking"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3","upstream_model_label":"Muse Spark 1.3 (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"400cab6060b4874691b6758f9243b07e3fcd6f11387789decb33b93d2ac62d9a","result":{"confidence_high":60.254348,"confidence_low":40.737306,"sample_count":97,"score":50.515464,"score_display":"50.5","source_stated_rank":null},"run_fingerprint":"7d2646d71d8b689419ed8a09983a7c0729122ca3c43c8499a572ee529acf7ded","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Max) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a3f9b293884e1fb51176e52c02e097eb0a860696109467fa556e7c6c643356d7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · tauBanking"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"muse-spark-1-3-xhigh","upstream_model_label":"Muse Spark 1.3 (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"50f0a26db16dfaa68cad067c39484f98639098fde28fdadea6c87348db1d9f7b","result":{"confidence_high":57.066997,"confidence_low":37.578071,"sample_count":97,"score":47.216495,"score_display":"47.2","source_stated_rank":null},"run_fingerprint":"d70fd4d2ef185a86fe7110982e6d305c4eeea2f67c185b8bf5a2dabfbe9d51ea","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"51059761b3695dee3b42d95796868843a7be4006e136217083b60818f23d89f6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.6990000000000001","mean_standard_error":"4.5","model_release_date":"2026-09-02","notes":null,"standard_error_points":550.0,"upstream_source_url":null},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.3","source_effort":null,"source_model_version":"muse-spark-1.3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"7bb591e7c5dda9cb2f317d6289fd05ac119b4c8ad509c41499271e4a63eaf5bf","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":57.8,"score_display":"57.8","source_stated_rank":13},"run_fingerprint":"578cdc8d5343a352e856a2ce001bfb9e63fc3f5e12a5c54b23bfe4c3ff878eab","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e01f91433fae684d4d2650c5","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"76a007f8d7b50ea922f9ace32caabf1ebf0b078504c6aed1a6623a11533e6589","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":64671},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Muse Spark 1.3 (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6ec54b101ae5cd0a277d28d846b8471fb94ba0ff335aa5a67a123722bfd67671","result":{"confidence_high":0.04593464296482699,"confidence_low":0.033982590688229,"sample_count":6139426,"score":0.039958616826527994,"score_display":"0.0400","source_stated_rank":18},"run_fingerprint":"730fee98a5afa26b4db7dc4043c504e7881c328935cc08af4b34b106ddc5e070","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"db282052ba0459d7c2a081c66dc79ad0928015023355b996f0ed56e15671536f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Muse Spark 1.3 (Max) · gdpval"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"muse-spark-1-3","upstream_model_label":"Muse Spark 1.3 (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"09973d8de41e10210c24d4a4c4909e6f150c8edce88315cc829d8df322e86e50","result":{"confidence_high":1694.25,"confidence_low":1650.4,"sample_count":220,"score":1672.33,"score_display":"1672","source_stated_rank":null},"run_fingerprint":"db0296662535d3eca737ec6cce88328906e68249c207d52037d7143f93130d77","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_942867dc5166c18b24f5834d","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e394d7983394b88609d45aee26b90c43f60ebd8e7dc3657f00192fd1083bc8d1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · gdpval"},"model":{"id":"meta/muse-spark-1-3","name":"Muse Spark 1.3","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"muse-spark-1-3-xhigh","upstream_model_label":"Muse Spark 1.3 (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"8574df1f65eabd7a25f0c77ee400b203a5da92ff0cab0e0c9ac5acda00822dd9","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1618.64,"score_display":"1619","source_stated_rank":null},"run_fingerprint":"1f48fca9331cacc70d0c01cc7067016671965c2b3b70a998076988f2b6f8e613","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.3 (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_78f4b55ed67e2456efcd3bf7","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"81d41db65e8a64f11667e70e5aacf9697d3c13c6d95cdf2713ab1a0a362cd0de","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-05"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.3 Codex; model release: 2026-02-05","source_accessibility":"API access","source_model_name":"GPT-5.3 Codex","source_model_release_date":"2026-02-05","source_model_versions":["gpt-5.3-codex","gpt-5.3-codex_high"],"source_reasoning_efforts":["high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"37f6011561f8f40c734581a8307c3439fe22cc4483eaf648bf4361de38a9bb5f","result":{"confidence_high":160.81,"confidence_low":153.5,"sample_count":null,"score":156.82,"score_display":"156.82","source_stated_rank":18},"run_fingerprint":"447f42d4cdf066a68694e5478ac7a35f9aad045f19e569a7d832ace17b20da04","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6fbb477e04d3bbc4452e0868","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"be678a0be67dc95300ef7b808ec98c894c32489f22b4e73fdba9b2857d8d04f6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":400000,"is_open_weights":false,"is_reasoning":true,"model_release_date":null,"source_locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · intelligenceIndex"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-3-codex","upstream_model_label":"GPT-5.3 Codex (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"8b4621755718fe6b93bb887dad5570c9d4a86e1c2ef4065551c8dd11cd2ba612","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.502817,"score_display":"32.5","source_stated_rank":null},"run_fingerprint":"0b14b1e5a3d576ca97984ebe49d976794b5632f781bd8f91570fa69a67047b10","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"db5028f902c6a22c54f00f17718d5d0e8f7e521deef8462ac64c20cab1fec4e3","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":400000,"is_open_weights":false,"is_reasoning":true,"model_release_date":null,"source_locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · hle"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-3-codex","upstream_model_label":"GPT-5.3 Codex (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2eda83ff0d78060e4a1a21b67939d9f65b3b5bbbbdddff8451a079a4b2a13b68","result":{"confidence_high":44.590266,"confidence_low":40.422512,"sample_count":2158,"score":42.493049,"score_display":"42.5","source_stated_rank":null},"run_fingerprint":"65a667fea5e80d6aea7bf0bf4e14f4868a6dee0605e4135e77f172205cec7327","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3d05a46ff4098f5768bf9b38c0bafcf9b2c063ebbd897c6a78f793ef32a7b065","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":400000,"is_open_weights":false,"is_reasoning":true,"model_release_date":null,"source_locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · gpqa"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-3-codex","upstream_model_label":"GPT-5.3 Codex (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d05a08b7d3d88d8d0f1f1c1d05ec24275150358ee21a9968d07d256489bf85e9","result":{"confidence_high":94.649684,"confidence_low":86.800325,"sample_count":198,"score":91.515152,"score_display":"91.5","source_stated_rank":null},"run_fingerprint":"ce66ef226abedb8a3fb035d4968dc5e37963ee3a8122b5ec1e36b0c9d5a6cba5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dfc28924cdad4bf64dbe74678b87af04e8b13d98a4549658d7a1961cf837fa8f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":400000,"is_open_weights":false,"is_reasoning":true,"model_release_date":null,"source_locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · mmmuPro"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-3-codex","upstream_model_label":"GPT-5.3 Codex (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2717d789c9f3d1ef5f90a9ac15108754e073c6901e33c666c818105df9d617de","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.49711,"score_display":"78.5","source_stated_rank":null},"run_fingerprint":"08de3f9963bf8a1b11ab31c661e7dc5a344cd89bbcdfb6b5894c1577a0769a77","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c6f673afa3bc558ec2919fc8aa9373e9bf09bc1672242332b2d2883f1caaeb25","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":400000,"is_open_weights":false,"is_reasoning":true,"model_release_date":null,"source_locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · critpt"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-5-3-codex","upstream_model_label":"GPT-5.3 Codex (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"572442f3bdadd49c7c3a6ba1f21f5bcaf8cb33cb236517908e2a1eed020320a4","result":{"confidence_high":27.2928,"confidence_low":9.869992,"sample_count":70,"score":16.857143,"score_display":"16.9","source_stated_rank":null},"run_fingerprint":"428da6cf775729138ad79e5d77aeac6fae2198559413753a062e2370ff80cacf","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.3 Codex (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"97db20223f89f2ef5bc8fa3bb65e7f911356561bd6434fe66367cfdb771de60b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.455384,"date_is_proxy":true,"latency_seconds":246.527,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.854},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.3-codex"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"ea6cfbeb3ada0b7b92cd2aaceef8326fd7d768abbecd61cac043891ffeb56022","result":{"confidence_high":81.63384,"confidence_low":74.36616,"sample_count":500,"score":78.0,"score_display":"78.0","source_stated_rank":33},"run_fingerprint":"07fe94d14ce2e9b23c15724508f6cd1d1104c8f3a1ae7b9ec3c2b04ec0b4d2db","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":222,"at":"2026-02-25T10:37:06.792Z","basis":"evaluation_started_at","evaluated_at":"2026-02-25T10:37:06.792Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"256c9f0f42f1c24a78721708e29306064ec1c67daa4ca6fa993e0e7a76d49f87","metric_details":{"best_score_across_scorers":"0.7479338842975206","model_release_date":"2026-02-05","stderr":1.9756739501473306},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6w5buMbgeEqtRoE55cLZVv","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6w5buMbgeEqtRoE55cLZVv.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/6w5buMbgeEqtRoE55cLZVv.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"af09111df0d000e3fd251aefe85fb41a93aa99af659a27dca87cc4c70b15718d","result":{"confidence_high":78.66570937204082,"confidence_low":70.92106748746329,"sample_count":484,"score":74.79338842975206,"score_display":"74.8","source_stated_rank":16},"run_fingerprint":"df01688549ddc48b5c85f64a49673316c09989567f6db7d2bca40a5d8c46934e","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_40e86d3b0dbb51ddc0083c6c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_7449132b5f078c71f23c26b5","split_or_window":"2026-02-15/2026-03-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":218,"at":"2026-03-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c91ea92cbdd81ab08dbb90a6bc67f5f9ed79361b29e0b6caf80d443f2432f751","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-02-05","pass_at_5":62.96296296296296,"potential_contamination":false,"sem":1.888525745775106},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"gpt-5.3-codex-xhigh__tools","window_from":"2026-02-15","window_status":"historical-post-release","window_to":"2026-03-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"cc0697656c8ec71762e5d7bf0b117c4a3e8ba600f39ebacc37d176dad21a65bd","result":{"confidence_high":57.775584535793286,"confidence_low":50.37256361235487,"sample_count":null,"score":54.074074074074076,"score_display":"54.07","source_stated_rank":null},"run_fingerprint":"8a083d3ebd38aeea5dfe675751088c1a6c0f4b403b2c2f99b64bc00096e49bf8","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_7449132b5f078c71f23c26b5","split_or_window":"2026-02-15/2026-03-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":218,"at":"2026-03-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b98a7c2ba4944430c8afaeae5c9d1afde2985896424857f0e7262ac6e34e4708","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-02-05","pass_at_5":53.84615384615385,"potential_contamination":false,"sem":0.942111439531991},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"gpt-5.3-codex__tools","window_from":"2026-02-15","window_status":"historical-post-release","window_to":"2026-03-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"48f8e5ebd6ec58cf25c64a21761226f7f72ca2844052e347da66db1c4b035ac7","result":{"confidence_high":53.38499995994424,"confidence_low":49.69192311697883,"sample_count":null,"score":51.53846153846153,"score_display":"51.54","source_stated_rank":null},"run_fingerprint":"c1a17abe81e2d5facb37ef2fd5a0ce69ba908d03d0bf97e05ab983fd81833644","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0463b4eba49654a3954857dd","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_b710408437ebab99c2a81296","split_or_window":"google-single-attempt-comparison","unit":"percent","version":"SWE-bench Pro Public · Google comparison table"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-19","status":"stale"},"measurement_id":"12bb666c50e1f09425a4b4b67bce4af6db33b4feed9ceaaf02d366f11362e8b0","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","evidence_sha256":"4c75baf400ffa3d11b2095ddd52c7fac9689fe3a545396e8f46d0fb0297ebd20","kind":"reviewed-static-report","published_at":"2026-02-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","version_id":"8280865cd0e38d4df4d25ebd0ff7d528ea0e07796dcba3ea1dea08b2faff6f9a"},"source_locator":"Model-card results table · SWE-Bench Pro (Public) · GPT-5.3-Codex Thinking (xhigh) column"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"xhigh reasoning; single attempt; Google re-report of the OpenAI provider result; exact task revision and sampling settings not disclosed","reasoning_effort":"xhigh","result_published_at":"2026-02-19","results_as_of":"2026-02","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"026e6cfdaf6ed476df3c435c09dabcd9e63ecfc1d0dd8c121f3fac7f581b0121","source_published_at":"2026-02-19","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Codex harness; exact version not disclosed in Google card","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"8ced76a0b13fd654d53267f0e966264f9859e0ca706e0ba18dedc371535092b4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.8,"score_display":"56.8","source_stated_rank":null},"run_fingerprint":"a3d1f853b7a198e05ad15043de25651675c53ea58db259b535cf9d3429a506a6","source":{"content_hash":"e6b36e17e4cfb89dc185360e76dc7116af66b9615badb20bc3b4ba6fd36baeb5","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","id":"google-gemini-3-1-pro-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card results table · SWE-Bench Pro (Public) · GPT-5.3-Codex Thinking (xhigh) column","name":"Google DeepMind · Gemini 3.1 Pro Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-1-pro/"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b7a0ba23f432b75a284778e2c059f974c8cbf6cad751fb0c0bc3dcda9f04dd7e","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.68083,"date_is_proxy":true,"latency_seconds":1791.685,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":6.312},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.3-codex"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"179bae66ccbd9ec167fb42096789261de2ad32f9b077d48a0b6a38d18095683d","result":{"confidence_high":66.20452,"confidence_low":41.461479999999995,"sample_count":null,"score":53.833,"score_display":"53.8","source_stated_rank":20},"run_fingerprint":"48eb5b9b61ad68ec4e27aee32e220b00a24cb08a7c5af664a7879043c3cd02ee","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3a87315ffec3f35c0b490d8d42e5a851962a9217e8e86f06ca1b32cb0ee46ee6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.091585,"date_is_proxy":true,"latency_seconds":132.631,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.968},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.3-codex"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"fcb812eab7708371e514931bf0d3bf09b5af27f963fed962dce313f726a5c496","result":{"confidence_high":89.21028,"confidence_low":85.41572000000001,"sample_count":null,"score":87.313,"score_display":"87.3","source_stated_rank":16},"run_fingerprint":"30f6f4da0e705556138a700cf796e8a4aef814d676ab694360772f01347ebb88","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"29067bd701a249d5fd8dc0487e1fbc761d204970598bc288e6b1ec2c43432d20","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.559265","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.3-codex-xhigh","source_effort":null,"source_model_version":"gpt-5.3-codex_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"a0120ce2bd373839706d8fa0190e509fac91b08265a22a625f7fc830c547805f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1655.22,"score_display":"1655.22","source_stated_rank":12},"run_fingerprint":"8aceace0efee23db77433bff217c905e47a400cb1826e047a99f2d4035167d15","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"37039b9331f1b61557acfe0ac01118b4a96f26567b9c615fbdba8b272f8d9820","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":11.914536,"date_is_proxy":true,"latency_seconds":4548.115,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.714},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.3-codex"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"fd4d7c92f1cbb6213fd50b36fb62810efe7708e5624db64cbfcca12ac9226e99","result":{"confidence_high":71.00644,"confidence_low":52.52756,"sample_count":null,"score":61.767,"score_display":"61.8","source_stated_rank":44},"run_fingerprint":"4f19d358890de977cbf1432e2eb9c21ca22f9e40ec5ce713a0b692d2a001f87e","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a77fe074fca6481009bf292b1db1ba6109b1c2a5a95134e6fdf4e9afefdbd692","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"63cd54966a4d22e4179a263f9ab2d1513969568cd7a5c895282a2190ced5fb2d","result":{"confidence_high":1381.39,"confidence_low":1358.34,"sample_count":3491,"score":1369.86,"score_display":"1369.86","source_stated_rank":97},"run_fingerprint":"8907b4dde0466fcc079f2f3228c851643ac2fbcd655a9c71702de2ab6d10d248","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_518da716906fb79fe2468157","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e8cb0eaa971030565099307c7cbf36bd6b2580e0b17f646aaa7933ac92d46160","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.3-codex_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"c82f5fb7bbb93fc7766d30ed54f98fab19151918fef609831c357b6139fabea9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.9,"score_display":"77.9","source_stated_rank":21},"run_fingerprint":"340f7384217cfedb5ae54bd693e57828b0708b234e3519df1ee84e88a09dc3e0","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6d0406c294d37b748a648034","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8dc80364c4112076077b66581825942f8335226516d30526650c863e87491afd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"811653eb508640e485c72786884f03b556fb3cfea34a0954bcbdcb47fa05b741","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.3,"score_display":"79.3","source_stated_rank":18},"run_fingerprint":"c6611cfc5e8215f4e263b892d4c2d1ff92546aa94a8691daa6613e38e0aadfa0","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_88c5bc5b6e52e6ebd14683a7","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":237,"at":"2026-02-10","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-10","status":"stale"},"measurement_id":"28e3896c0502d3b1cdfdfbd6085ac6e8ab3acbd162e32c4102ecfdf79a89c86c","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.6,"upstream_source_url":""},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-10","source_scaffold":"CodeBrain-1","upstream_leaderboard_url":null},"run_count":null,"scaffold":"CodeBrain-1","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"00ba39be91ce2d03e38899bdbfc4f065bce89cf45528af4525b446c855e9ad3e","result":{"confidence_high":75.396,"confidence_low":65.204,"sample_count":89,"score":70.3,"score_display":"70.3","source_stated_rank":24},"run_fingerprint":"e481c836cb5d7fe918972759e17a2b633ee63f6e414f1e7e22048297a448dad4","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"11d09ddb80d0cd24daf4d18540fcaa7ba4818dfcf8b9d3522ebf08718f312198","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.202247191,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-05","source_scaffold":"SageAgent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"SageAgent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"59008774ae9f080bec380e735c2c632b5efda89f09f8bc19763515f6badc0c9e","result":{"confidence_high":82.74337078646,"confidence_low":74.11056179774,"sample_count":89,"score":78.4269662921,"score_display":"78.4","source_stated_rank":9},"run_fingerprint":"c25b5e83195f734cb55e800641eefc37847fc9de088a59353a41d9828ac243a2","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"7598951a33f12861efbf89705008e75bd48326d186a4b556791fb84136c6cf2f","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.1577527622,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-05","source_scaffold":"Droid","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Droid","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"f3de9e5231be947cef6a84276382fa752a6252063f06feef10dc57c9d5adf3bb","result":{"confidence_high":81.53256620041199,"confidence_low":73.074175372588,"sample_count":89,"score":77.3033707865,"score_display":"77.3","source_stated_rank":12},"run_fingerprint":"752260c1077c66e87999a411857b1fba6d70b29411f336399815012d7c879083","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"a53d36311e4c20271695b893b3a1a3ea4407837d1f05a256063612b282b854d5","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.0303715883,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-05","source_scaffold":"CodeBrain-1.5","upstream_leaderboard_url":null},"run_count":null,"scaffold":"CodeBrain-1.5","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"8960d21ff7de46a478c38c252ef84023b402317bd9bfcfb733612bf4f063180f","result":{"confidence_high":79.82222494226801,"confidence_low":71.863168316132,"sample_count":89,"score":75.8426966292,"score_display":"75.8","source_stated_rank":14},"run_fingerprint":"2b5be58e7841a5fca8d4ce98d1c44397bc447ebeb1b461e3d87fd5810f174242","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"30275a1ca63dd1d0c7074274575012b9d885e53b342de37431f11974dd18b5b0","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.1577527622,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-05","source_scaffold":"Codelia","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Codelia","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"3903b4be4b8c319bf138fceafd066e01c1344d2c46780eb96fb6f9d250f96340","result":{"confidence_high":79.959532492612,"confidence_low":71.50114166478801,"sample_count":89,"score":75.7303370787,"score_display":"75.7","source_stated_rank":15},"run_fingerprint":"ae1933b17b60b98662b5b1b18a8bf51ff3f50c461b01b569422b44c0e2f38ee8","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"3a69c41dcd111afbde2885d1a0698da772501e44c6e100c36dc62e291e9047e0","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.3922527823,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-05","source_scaffold":"Simple Codex","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Simple Codex","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"9b2a589c7427eaba7c631c8f2b8d1a2f28e294d98da5f9a7b2d6332a277c6f94","result":{"confidence_high":79.74499522860799,"confidence_low":70.367364321992,"sample_count":89,"score":75.0561797753,"score_display":"75.1","source_stated_rank":17},"run_fingerprint":"5431e0e405657601aa472bca52fb924df1cc18055712d2011e00b4a81b03795f","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"f44c190ddf21785b27a9898bc989bbfc34e31a273f6a348bcee3d7a458c072b7","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.5109481272,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-05","source_scaffold":"Mux","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Mux","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"d0927a914e5a434685865da3246a8b21e82ae74449e931ec9e8d7a70ad718cc5","result":{"confidence_high":79.528199902312,"confidence_low":69.68528324368799,"sample_count":89,"score":74.606741573,"score_display":"74.6","source_stated_rank":20},"run_fingerprint":"04ed79532e3304d3a51bfe6bce1e0d7bef2628c81a54312b6a95b5e59ce5f3fd","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"9062b14f983cb6978e1f363b80c4c3585b52e8c30e149ecc0521f76319382e61","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.4915582762,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-05","source_scaffold":"spoox-o-m","upstream_leaderboard_url":null},"run_count":null,"scaffold":"spoox-o-m","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"912f4286fa2f7d184da3c3cccadc9cff42d90dbe120810d21f270051805c0bd6","result":{"confidence_high":76.344128378652,"confidence_low":66.57721993594801,"sample_count":89,"score":71.4606741573,"score_display":"71.5","source_stated_rank":23},"run_fingerprint":"2fe235ab0cb0c254e000d9d3a1f6b669a1c5fcf40b4ae51cd2a5c3b0a3d9667e","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"d08da24f882f8e12c4a530cb3d724a9be7d592671022ed20a8b3ca16914314a7","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.2792046745,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-05","source_scaffold":"IndusAGI Coding Agent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"IndusAGI Coding Agent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"24fd198b76e8afa87bdda09c9fa84c86e425295fd60a4d6da1c30d918e5142fd","result":{"confidence_high":73.60581794102,"confidence_low":64.67133561698,"sample_count":89,"score":69.138576779,"score_display":"69.1","source_stated_rank":28},"run_fingerprint":"fcd67e23a175b64501246ef388d44c7097c79aa5fad4415fb7fd70cb2c3fcb88","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"1e3313867f7157ed8c3ea316ed78d2f649a77ed5d02d63230cfa8bf975569500","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.7151126845,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":"2026-02-05","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"3867b01c2907778f6d2df3b27909bc021f664a3e92f7965171fc994ca67d8d00","result":{"confidence_high":70.04072198522,"confidence_low":59.39748026198,"sample_count":89,"score":64.7191011236,"score_display":"64.7","source_stated_rank":36},"run_fingerprint":"94071fcc98f9aedf844ad6da9b636f9df3f35b4b02738a9a161d1ee807835b6d","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49c3888e1ad7bc9e55b2b6e3","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"7c32643bfaa2033f22d02e018b81588d991e58a0672c7c22fccc9265ae719c2d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.469","mean_standard_error":"3.3","model_release_date":"2026-02-05","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.6,"upstream_source_url":null},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5.3 Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"c954592480c137b6008ccc3e5ef2d21c5f95bbf3fa7a70efb3e1651a7124530b","result":{"confidence_high":38.856,"confidence_low":24.744,"sample_count":480,"score":31.8,"score_display":"31.8","source_stated_rank":26},"run_fingerprint":"078c275ac63a8b8cafcbf0f86acfc409274d718a778049385d52b53ab60e1b0c","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c994fb9d010b7fd17fa655bb","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a21e396336f312d812b892c04d3a493f0ca15821f724eeeef3bc1c820ccc2b36","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-3-codex","name":"GPT 5.3 Codex","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.3-Codex","source_effort":null,"source_model_version":"gpt-5.3-codex","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"a29646b22fed99982e634b5dc1a452c231c8ab01f4da23531956ae6b62aca732","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5940.124000000002,"score_display":"5940.12","source_stated_rank":18},"run_fingerprint":"ad9f6c48c0a245f7857e59fc82fe801aaa1d2cc3325c1e08178b5e64c324436d","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8a765e2735eab6c66f923409","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0ed68ae0eb24ed9fbcb3ba4bd2190fca2b81fc5474109c59dfd7f4a66a4cee97","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-08-12"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Grok 4.6; model release: 2026-08-12","source_accessibility":"API access","source_model_name":"Grok 4.6","source_model_release_date":"2026-08-12","source_model_versions":["grok-4.6_high","grok-4.6_medium","grok-4.6_unknown","grok-4.6_xhigh"],"source_reasoning_efforts":["medium","high","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"df7bb7282f62eba39a22e314410ab65734585f0fbb3bd93b4dc37d26eea11db6","result":{"confidence_high":158.93,"confidence_low":154.66,"sample_count":null,"score":156.61,"score_display":"156.61","source_stated_rank":19},"run_fingerprint":"1220f2b8f6b9665a404194af3508ddec61af378fc0c6a750acb712f3076d941a","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2c4fefa8eb830ac7d5d0c014","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"383493b2f2863331fa2fd1e5ac5a088be35795d163fdbafc79c2854a321ff0ec","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (xhigh) · intelligenceIndex"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-xhigh","upstream_model_label":"Grok 4.6 (xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"ce295e45d42c5d782c6e8d2e14512d1aa220203d68d3528e9f704bc5722e9e1a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.27016,"score_display":"44.3","source_stated_rank":null},"run_fingerprint":"073cec4e789330c6ff090d1af2255a46c0bbbfeaa1b7808b99d867c03655f254","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9fdf8658822e0cc95e0ed5deb546288e81f96258087a3e1a2d950f7701057b2d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (high) · intelligenceIndex"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6","upstream_model_label":"Grok 4.6 (high)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"a3c94e3b640b85342ce87870ab18e276b101b5090837024784a7c1b9c02afde6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.405007,"score_display":"44.4","source_stated_rank":null},"run_fingerprint":"d67c63cfa2f664a48121af968f51801ba0dce1f95c2caa7a5824d2ce3a205f3d","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (high) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"04ff99b8d4c2b3919e25872d5f560a984a4ba1786bffaaef1159d5030bd94613","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (medium) · intelligenceIndex"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-medium","upstream_model_label":"Grok 4.6 (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"120fbf63893f40a0ebcf94c0878730be7ed53c8185d2e7275deec844704ac31c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.011173,"score_display":"43.0","source_stated_rank":null},"run_fingerprint":"8414f0a7544a96a06d54e27455ce7f8aff479d093715095b06c54cba00f9d5ab","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bc4febb7b1276abf8662a16c23397762f34337007462c4eb569cfe9cda9df3b0","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (low) · intelligenceIndex"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-low","upstream_model_label":"Grok 4.6 (low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"0c80d5544e3b17e0acaf6b588fa1aeceaef74ebf2f44b616e963dac9a2ca77c6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.414845,"score_display":"35.4","source_stated_rank":null},"run_fingerprint":"7041c93d8612ab82621556a5d036926bbf14f827d2892ab078704d6d28d72de6","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b3a0aa205f2111fb2dfc2cc10f0ca38c5c43f300eacc083e7c11ba26088d32ee","metric_details":{"license":"Proprietary","variance":6.830383057793102},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1458.9486202515975,"confidence_low":1448.7038878986118,"sample_count":23680,"score":1453.8262540751045,"score_display":"1454","source_stated_rank":73},"run_fingerprint":"c76e861a8b8ea292589d290e86577921ab105dbf18e3175645efd396bbd90979","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4165729358a384ef84d3fa7a8139789810d45ce4ffbc22fa3cae6bc44ec0a9c6","metric_details":{"category_scores":{"Agentic Coding":57.02,"Coding":76.77600000000001,"Data Analysis":73.857,"IF":71.8665,"Language":83.697,"Mathematics":92.56799999999998,"Reasoning":90.5095}},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.042,"score_display":"78.04","source_stated_rank":18},"run_fingerprint":"51b43b4ecc06f1a89002a9df24335ee2c3e881a5a91687c8e723870cb5d99050","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_16f1405d00db208b1c4a51a1","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"205fc5c86d86307baf14f99cb452d24b09b802cc6edfea1360227f0f0193b000","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (xhigh) · hle"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-xhigh","upstream_model_label":"Grok 4.6 (xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"cfdbd8716d06d4eedce197dbfe61da5d07716d5493fde410304ed39f2f4a3e62","result":{"confidence_high":46.171991,"confidence_low":41.986253,"sample_count":2158,"score":44.068582,"score_display":"44.1","source_stated_rank":null},"run_fingerprint":"c90adc67a939d048c185dd19ba8c3a7c17678fb4c75f67e58339d39270de659b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6b0949c5732f75fb9efc92127b7587478353f745d9dee5d3731d92d1027d6d1e","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (high) · hle"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6","upstream_model_label":"Grok 4.6 (high)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8645a7f92e2909d54d65e81f8428345261d538c4b9fdb4f78fa828a02cdc7585","result":{"confidence_high":45.009167,"confidence_low":40.836234,"sample_count":2158,"score":42.910102,"score_display":"42.9","source_stated_rank":null},"run_fingerprint":"77ba2f53f105ba8c04cf5f73021227145b093f7f06176ce425d27ebcea254290","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (high) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4eedf562f4b4466fcc22edcd410239faec45b34f73414e381c7ac06d51625c69","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (medium) · hle"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-medium","upstream_model_label":"Grok 4.6 (medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"bf58888535bffd8a01727ae9f00d6cdd728f25f5299e947603130947681b00f2","result":{"confidence_high":44.217782,"confidence_low":40.054885,"sample_count":2158,"score":42.122335,"score_display":"42.1","source_stated_rank":null},"run_fingerprint":"b001ef375d5dfabf189b8867b8b1ebc17b5d51355c96062d8a28ca7a66706dd2","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"843531aa1fc729fb9abddb7db4694a1e6ce359a85cad1243a2dbce073658b078","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (low) · hle"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-low","upstream_model_label":"Grok 4.6 (low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fcd0e354e54a330a9d507b62090997ae6574fbd385d62e2c539aac84cada6d9e","result":{"confidence_high":29.543118,"confidence_low":25.772757,"sample_count":2158,"score":27.618165,"score_display":"27.6","source_stated_rank":null},"run_fingerprint":"d4b750afde9ddaec258f04e4025a57f98608353ab6dbc1da2932585220c3dfe3","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:29:19.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:29:19.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"48682767c901ad37a52d78bd5318e4d875b88b5c7bf8576833b0e6c1e0e95302","metric_details":{"best_score_across_scorers":"0.489","model_release_date":"2026-08-12","stderr":1.5815471195292574},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dKCWFaoGCWJTnD5CPWQGru","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdKCWFaoGCWJTnD5CPWQGru.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/dKCWFaoGCWJTnD5CPWQGru.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"da916f7fcb980502d9849f78cb0d5296fe195cadca371274fe57d0a51007ab2b","result":{"confidence_high":51.999832354277345,"confidence_low":45.80016764572265,"sample_count":1000,"score":48.9,"score_display":"48.9","source_stated_rank":28},"run_fingerprint":"0b7605ebc5b49403ac202133cc11bc2779c9d8f500a09fbfbccb59de9c78011e","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:29:53.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:29:53.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"10445bf5e5ec5b6db3cdeb1a30d52b4f8a2aebecb37786451ad7795169923649","metric_details":{"best_score_across_scorers":"0.493","model_release_date":"2026-08-12","stderr":1.5817749561843528},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"TpHYtuPG6kYZFvMK7Z6Hqj","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTpHYtuPG6kYZFvMK7Z6Hqj.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/TpHYtuPG6kYZFvMK7Z6Hqj.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"050637a1e787dde7eb40faf50db87b07a7898e64932b008b2044d146148fc756","result":{"confidence_high":52.40027891412133,"confidence_low":46.199721085878664,"sample_count":1000,"score":49.3,"score_display":"49.3","source_stated_rank":27},"run_fingerprint":"5ee068b95531ddeb138a59f0cfafbfe28f7568e08811cc029394363bdf3c9f05","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e9e0d35a1eb01b345dbdd6325ad40e3f471e2a1a7808d27e8156d84d75f05832","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.045365,"date_is_proxy":true,"latency_seconds":209.056,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.125},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":93.939,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":null,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.6","zero_shot_accuracy":95.455},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"3f48fbac233c647c905a65f76defdf1b5c11c2c020674e511ab9c25cf60863ad","result":{"confidence_high":96.902,"confidence_low":92.492,"sample_count":396,"score":94.697,"score_display":"94.70","source_stated_rank":3},"run_fingerprint":"1f42197d0765bf574e01dff5d5e03250d6015988ebf2bc91a8d1a0ccfdfbf824","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ad2dadce5ab23f84ac828923f8449a20811f84bb45d2f32dc39bd96e61b3d8f9","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (xhigh) · gpqa"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-xhigh","upstream_model_label":"Grok 4.6 (xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"df1281424a3d0d3992ded14a4ba6156954f060b4182dc6018722ae80dda4b620","result":{"confidence_high":96.198914,"confidence_low":89.2146,"sample_count":198,"score":93.535354,"score_display":"93.5","source_stated_rank":null},"run_fingerprint":"0a259410e567741e52616454b535396e0f1b3c21af05301aca3277b278dc3580","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fa45553a1a6322b6cc21b62e86fceecf265ec85e4b7be1400567c80e599ce436","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (high) · gpqa"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6","upstream_model_label":"Grok 4.6 (high)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7bf479ebd5c700d928e4c81da9c81aa712689e0a7844fd529823a0c3968627b1","result":{"confidence_high":97.233874,"confidence_low":90.954091,"sample_count":198,"score":94.949495,"score_display":"94.9","source_stated_rank":null},"run_fingerprint":"25bc2c132016710e249c349eb5e2e25b6dde0d60c64aba7262565fd1335b3f6a","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (high) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fb86e22c58e9863e98dba90950aed203025ae01a64193ee458706f406ba96f52","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (medium) · gpqa"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-medium","upstream_model_label":"Grok 4.6 (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a8b8d444a14be6f01b5eca942e43a214c110459f509da8d93dab1770091d7c7e","result":{"confidence_high":96.198914,"confidence_low":89.2146,"sample_count":198,"score":93.535354,"score_display":"93.5","source_stated_rank":null},"run_fingerprint":"7e51816b032ba5cbd77b6f0c978ecf28e57c437abec457c620af6a1d357791ed","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"35ceed990ff8937e21845c8b9091d7c520d441c1c619b0a72f635154a3b25167","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (low) · gpqa"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-low","upstream_model_label":"Grok 4.6 (low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"350abbbbf32db48691b7f036ad2530c1fd6003c1ca9db35c7627c5c333707030","result":{"confidence_high":91.717827,"confidence_low":82.597874,"sample_count":198,"score":87.878788,"score_display":"87.9","source_stated_rank":null},"run_fingerprint":"3636552d1b4a5f72ed0f1910be45725a81908c54c050f594d60bfd4573aa1d89","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14T13:47:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T13:47:42.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7e5d076278f999eb67086c5c0867042fcab0848fe0769bc46c9e19a86f77eda0","metric_details":{"best_score_across_scorers":"0.9318181818181818","model_release_date":"2026-08-12","stderr":1.5204299687218694},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"9YJf2nviwjM6WVhw8qeJta","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"50106d40a1b0f97d7ad9f7a0c9839b43b42f8286307a81f1f8beb3a9041cf56d","result":{"confidence_high":96.16186092051304,"confidence_low":90.20177544312331,"sample_count":null,"score":93.18181818181817,"score_display":"93.2","source_stated_rank":19},"run_fingerprint":"03cf3accb6558ce66a106f82919974e610411ab700d8ec317595eb4c765daa87","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":53,"at":"2026-08-12T19:39:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-12T19:39:52.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f6cfce69922decd5d11048d05b7c1218be51e2c681abb4c6fa652e46f02a8a0e","metric_details":{"best_score_across_scorers":"0.9400252525252525","model_release_date":"2026-08-12","stderr":1.4480018812911388},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hnP6Z88x8bPEBRZ2PW6kps","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"bdbd50bc3818b07dad99b8d409020a8f94b1c069be549059928a990b0bb76da5","result":{"confidence_high":96.84060893985588,"confidence_low":91.16444156519461,"sample_count":null,"score":94.00252525252525,"score_display":"94.0","source_stated_rank":11},"run_fingerprint":"f22cbd7008ee44b70d6076252d1238961074708b74a19853726b1ff9da67ad1c","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a9615deec467c36b75a2c484c8eb37f6d3ee7c273e4b78dc16c1ec728d461841","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.015391,"date_is_proxy":true,"latency_seconds":54.975,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.305},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":null,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.6"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c9d551bcb12eb542e9f4533ce5bd0f77689cad2d1202652d312ce29f723a557c","result":{"confidence_high":89.99780000000001,"confidence_low":88.8022,"sample_count":null,"score":89.4,"score_display":"89.4","source_stated_rank":11},"run_fingerprint":"5d06480895353430b65f14e8e3bf8b655d8b967af89fbf12f781904e4b2f79d9","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":52,"at":"2026-08-14T13:47:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T13:47:42.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"13acef94aed9fd4a0d5de32baaf49fb9677739980ae696798f2fca4288593d27","metric_details":{"best_score_across_scorers":"0.9916666666666667","model_release_date":"2026-08-12","stderr":0.4700633956814717},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7crLvnfQc987dC6mr3iGbf","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c93bc277c165a94c5c0d5cc658ffaf45c61cac8abd5fae1685dd16212920b64e","result":{"confidence_high":100.0,"confidence_low":98.24534241113099,"sample_count":45,"score":99.16666666666667,"score_display":"99.2","source_stated_rank":15},"run_fingerprint":"b970054d1bd61b0de37bffdbafc4762838d793083cdee06791b71d826aaf7f3d","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T19:39:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-12T19:39:52.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a469f342e7d5f0251ce2293b4865103db30d76701806b4afc3cc6e40418f918e","metric_details":{"best_score_across_scorers":"0.9777777777777777","model_release_date":"2026-08-12","stderr":0.996275330534286},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jtfdCURnwfZzak4BSiBKzx","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"100c9cc06cf2df774c716eb5ed0d84c8ca01f6f629094f096ca4c0f01adbd40d","result":{"confidence_high":99.73047742562497,"confidence_low":95.82507812993057,"sample_count":45,"score":97.77777777777777,"score_display":"97.8","source_stated_rank":25},"run_fingerprint":"c4654e15f5c8b47c1ab36882f79120a8d9ce051a0902b6f436be729dc7665d0e","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":51,"at":"2026-08-14T15:46:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T15:46:50.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"95ed029b581be7234ceb25529620f1a5bd487877ae717b2948a017090af6ba48","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.6596491228070176","leaderboard_private_problem_count":285,"model_release_date":"2026-08-12","public_example_count":10,"stderr":2.8116467881852296},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"Zb2AKwa942D7dBTTcebAba","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZb2AKwa942D7dBTTcebAba.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Zb2AKwa942D7dBTTcebAba.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Grok 4.6 (xhigh)","thinking":null,"upstream_model_id":"grok-4.6_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"1d70122469f92bc372a5aa8789c446f32984b20e1520f06417885ade1646b058","result":{"confidence_high":71.4757399855448,"confidence_low":60.4540845758587,"sample_count":285,"score":65.96491228070175,"score_display":"66.0","source_stated_rank":28},"run_fingerprint":"f9a9e92d70c8c25f3cbd6511f23bb9dd9cb766f5499e8b6349940fb478aa3366","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"226913ee5bdcfc845ea84927d0b9857e7e140054bef80f2c02360d98a086616e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.7574904499999999,"model_release_date":"2026-08-11T00:00:00.000Z","results_url":"https://arcprize.org/results/xai-grok-4-6"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"xai-grok-4-6","model_type":"CoT","upstream_model_id":"xai-grok-4-6-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f2deb1a5fb373735d8e5af609ba2b55d428ea3ed188e7920f86f3a3b19d1be5c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.08333333333333,"score_display":"67","source_stated_rank":61},"run_fingerprint":"d92905ff88d99c3404c1ab4d2d19e3f7fab50720d504bfcce0f257a01ae4eab8","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4f56fafeb05d15e9bf8220e094d0c1e09495b96d06e48c9d12d33682f863e0c1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.7338575666666665,"model_release_date":"2026-08-11T00:00:00.000Z","results_url":"https://arcprize.org/results/xai-grok-4-6"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"xai-grok-4-6","model_type":"CoT","upstream_model_id":"xai-grok-4-6-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7bafa23037a381500e460f1f560619f04f09872f8be1490b1967263c92c4bc42","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.13888888888889,"score_display":"65","source_stated_rank":67},"run_fingerprint":"ad2c8f00438794214c937455c07963d6def295731c7d4d1794ceea12623347f1","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"083d48ebf9a309d0088078c32d02e456b3fb469967b856d8f40b907241af0e29","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.6586968666666668,"model_release_date":"2026-08-11T00:00:00.000Z","results_url":"https://arcprize.org/results/xai-grok-4-6"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"xai-grok-4-6","model_type":"CoT","upstream_model_id":"xai-grok-4-6-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"aa3ec35f6067ab1c55c5b14295cc16b42918b2d89788a11042b7cad565215033","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.25000000000001,"score_display":"61","source_stated_rank":73},"run_fingerprint":"c1cf3e0f8bec1d6826c67e40b843dda8c46ce25eec6545178e8924e9aba284e6","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5df950cdd421f0619ecc7e96bb8e28a5ac3a2d977a7de158cb6ae0afaa22c2e9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.2124022166666667,"model_release_date":"2026-08-11T00:00:00.000Z","results_url":"https://arcprize.org/results/xai-grok-4-6"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"xai-grok-4-6","model_type":"CoT","upstream_model_id":"xai-grok-4-6-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"55009c2ab24df452e7f6cde7ebe667fd89bb5e8b37962081216507e67047284c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.63888888888889,"score_display":"28","source_stated_rank":124},"run_fingerprint":"94c0d321587949a515931561baafe8d1903c066b6e9201d03f748edcc79348e6","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6b79c470e7a52f0e7360981e40430cf6678bb3cf843953eb602928ad9312e96c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (XHigh)","source_effort":null,"source_model_version":"grok-4.6_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"df34cca06c69695b8516559b1fd235f910bbb6122359a5f4d409f2494e6beda1","result":{"confidence_high":71.73486,"confidence_low":62.07106,"sample_count":360,"score":67.08333333333333,"score_display":"67.1","source_stated_rank":65},"run_fingerprint":"1da65defbfe90c8b73a22e3be5de95321d71090605779c97c9cc1b7c3022cf19","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7e8451eaae3ff79db50e69144cc2605dd534efa73521e37f4ef9bda13d137719","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (High)","source_effort":null,"source_model_version":"grok-4.6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"97f3da9e1b8cc879e71c0f2b589c98327c0d3bb1496488cc042673ada9a88f54","result":{"confidence_high":69.87821,"confidence_low":60.079882,"sample_count":360,"score":65.13888888888889,"score_display":"65.1","source_stated_rank":70},"run_fingerprint":"324bf659468c11d0e744e7b5ee5c82e5a6c7f65195056d699384856782a5a4eb","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d571a7a1b29b49689b22da4d3fc8d0e1791b34a7a485696443783ddb19847775","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (Medium)","source_effort":null,"source_model_version":"grok-4.6_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"90adc268b241e30a5fae20e6d7b59d430c8e3f87eedb05efa4b939460a1e9ee8","result":{"confidence_high":66.138603,"confidence_low":56.123832,"sample_count":360,"score":61.25000000000001,"score_display":"61.3","source_stated_rank":76},"run_fingerprint":"8805f1f863bdc616092488346651571179a3bfc24b6743fb2b8b774de86d67ec","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"337e41a0dd6d2fb3e11732548327562281ed553c99853bea35a842e3b162dbff","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (Low)","source_effort":null,"source_model_version":"grok-4.6_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6e035ee94da32b22bc1ce1b8280a5dada22d6fa65f96012d89687588f3fe4650","result":{"confidence_high":32.476337,"confidence_low":23.273638,"sample_count":360,"score":27.63888888888889,"score_display":"27.6","source_stated_rank":121},"run_fingerprint":"44c883d545a1c4774603e56d7a739e01fe814f52256878496e16b0e72dac4a22","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":51,"at":"2026-08-14T15:46:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T15:46:50.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9341982de537533a0e466b994878745a511f74dc4183f6aad65febd84f74b74c","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.3170731707317073","leaderboard_private_problem_count":41,"model_release_date":"2026-08-12","public_example_count":2,"stderr":7.3576112824382225},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"bg3JRdd33iWfvRoGSvyXXE","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fbg3JRdd33iWfvRoGSvyXXE.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bg3JRdd33iWfvRoGSvyXXE.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Grok 4.6 (xhigh)","thinking":null,"upstream_model_id":"grok-4.6_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"6ed3f071c5e7750e1f2076ddd705f98a41b812db3c402e1f7fec0f3ba3bb28e3","result":{"confidence_high":46.12823518674965,"confidence_low":17.286398959591814,"sample_count":41,"score":31.70731707317073,"score_display":"31.7","source_stated_rank":31},"run_fingerprint":"23f53d0529e9b4cfb279ec4d812dc0d0eb7687efda49c528eecf181b99e68027","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"abd798a6c8be7ac134c44b7088def8f4634677f07dd5f5b6f6cf80b252caf926","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (xhigh)","source_effort":null,"source_model_version":"grok-4.6_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6cbe59194a687095e35cb22830118e909d8f329a911ae3426cb497c183fa085c","result":{"confidence_high":30.415434,"confidence_low":12.122254,"sample_count":71,"score":19.7142857142857,"score_display":"19.7","source_stated_rank":54},"run_fingerprint":"f16ac401d9f4e87775b9afc1f7815dfa2d7541a956f1ce04eee49bcce84fa012","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"013d77f3aed6f9cbd8397bea5ee05e8183abc83b2b8ab9d555039b54779fac1b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (high)","source_effort":null,"source_model_version":"grok-4.6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6c248ca0a32343576dd6845c47dace7e31e328deddb681e649afb59c1b605ca0","result":{"confidence_high":27.533062,"confidence_low":10.12575,"sample_count":71,"score":17.1428571428571,"score_display":"17.1","source_stated_rank":69},"run_fingerprint":"aa8ad787ea61613f78e12f6e09badf7e58683202374f01efe71c4f8d4139667b","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1ad33568fbced3a03a37b83bb98fd1313698d4ae284a122e8030a5840d5a6f55","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (medium)","source_effort":null,"source_model_version":"grok-4.6_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a16bbc143c737bb07cc0875575ae80417e6c77a1a502bb2252b4ec75b7f2755f","result":{"confidence_high":28.178675,"confidence_low":10.564331,"sample_count":71,"score":17.714285714285698,"score_display":"17.7","source_stated_rank":61},"run_fingerprint":"92cd45cfa98b6a599e85d58420e1a028d7b612ff4c8698447a686d1518a71b53","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"22cd6616a574c61120cceb082b0a8f5b308525156c800c0123b882d7ed5acdeb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (low)","source_effort":null,"source_model_version":"grok-4.6_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"486c4ff419015ebad5c11ebba65d0dfd6368e6e55acce8bcf2755a48ac826600","result":{"confidence_high":13.716555,"confidence_low":2.258365,"sample_count":71,"score":5.71428571428571,"score_display":"5.7","source_stated_rank":106},"run_fingerprint":"5a4951b80d34931a433025036eed7e3f57754461fc9dfdfa8ce05ae35d853be5","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"23a121d89c554fdb8ed4be09abb0f2a7b4a0fdaf99773b1342ca7c3e7c3062ea","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (xhigh) · critpt"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-xhigh","upstream_model_label":"Grok 4.6 (xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"f1ec7a4363e76eba819c177fdfd9f9846515c1da00a8348613a570874d98d1e2","result":{"confidence_high":30.500013,"confidence_low":12.07978,"sample_count":70,"score":19.714286,"score_display":"19.7","source_stated_rank":null},"run_fingerprint":"63c77a31275ff4c5d6143b65435670dc8052caa32e23b57e1dfe43fa71fe48d4","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2d99e361194571d3da42e545b5b106793138bd0b908185ea7ea05418d48254c5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (high) · critpt"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6","upstream_model_label":"Grok 4.6 (high)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a0e44b61a010c875eb270cd4e51919fc053c2703f7dd51f16a5811a978739af3","result":{"confidence_high":27.616866,"confidence_low":10.087626,"sample_count":70,"score":17.142857,"score_display":"17.1","source_stated_rank":null},"run_fingerprint":"2cc5722d28ccbeaf0c23a0c171d763fd187799004749e5eb16f395abeec1174f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (high) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6e9af4f63d7b0116c07a4f41b66d97da77e7d663d0427a0371b70a83c6b90025","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (medium) · critpt"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-medium","upstream_model_label":"Grok 4.6 (medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"ccbde9204d5f4b2d1e64296b757263a600f27aa347c4e9c115d64a812c5e3b0e","result":{"confidence_high":28.262679,"confidence_low":10.525214,"sample_count":70,"score":17.714286,"score_display":"17.7","source_stated_rank":null},"run_fingerprint":"466cdd61201e2e4f9ad3e97c7a6d020ccada3551e9bfa3aa6025f504024a1f82","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c79062e4a02d4b72a47616e06c1707b47d33fdf01a1aa0e66430ad0d839f6cc5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (low) · critpt"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-low","upstream_model_label":"Grok 4.6 (low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"2fdc92896bc86f3dd9784f46a3eb530547f9dafc16c5647ce26b41c1fef9b3d7","result":{"confidence_high":13.792148,"confidence_low":2.244342,"sample_count":70,"score":5.714286,"score_display":"5.7","source_stated_rank":null},"run_fingerprint":"8ddcef2d4a3d2571bf7f4f44144051555ddfa58c849e68c72c97ba5f9539eac2","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e9cdaa438ed3a848c85be9f2f88606e40e060f63db4bdcfe974ee26b2dc73357","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4.6_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"6333f90af491572b991ae0a443247ffa5468890e9143e9ab7c4cd0c3fff81cfa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.9,"score_display":"75.9","source_stated_rank":8},"run_fingerprint":"bf2bdd768572a046a594939a0abd784d55a63ddcd798299c63976773f220887d","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9702996790d011ea744f76f7","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":52,"at":"2026-08-14T13:47:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T13:47:42.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6a9498f96b151d82accd55e66034cde1fb59d3cf236a40e288f5f40001d000b9","metric_details":{"best_score_across_scorers":"0.31","model_release_date":"2026-08-12","stderr":4.648231987117317},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"RiyrdXiqrXT98mKM9nMsmu","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9114b2e5a3c263ce24c0849f7e61f052dbc82d38d1da81f7b5f14649f387ad3a","result":{"confidence_high":40.110534694749944,"confidence_low":21.88946530525006,"sample_count":100,"score":31.0,"score_display":"31.0","source_stated_rank":48},"run_fingerprint":"b3a3786f840470ec644244a4bd7a1800c75fd8bcc3d05d27570eb39c16cea509","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T19:39:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-12T19:39:52.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2b8d1675be37fc0640080f37fc2e522b98ca3c5c18788fecc926b0157987e990","metric_details":{"best_score_across_scorers":"0.4","model_release_date":"2026-08-12","stderr":4.92365963917331},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"NGQ4Q8Wrb4aroMu5RecTS6","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"cbb63d3ae7c72511ffc8925b5a73445d65541294a03b920ec7764a45820014ca","result":{"confidence_high":49.650372892779686,"confidence_low":30.349627107220314,"sample_count":100,"score":40.0,"score_display":"40.0","source_stated_rank":24},"run_fingerprint":"4d65ccc830656a48434ddabe3947678558a3225fd0b7677b4b0c827d60cb30ef","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"05dcd04436b1c58512e7d92c4344e277acde7e811ea05c3f4d1d6b2a99688d18","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (XHigh)","source_effort":null,"source_model_version":"grok-4.6_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"67febf66d31d5521c9eb0190150dfdad3a229e9ce81cef5873103d59a6c61678","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.0,"score_display":"87.0","source_stated_rank":89},"run_fingerprint":"e76debed417c01c2b4607657ac1186fc0c2e63d1e2a1115eafecaf9a21a27e3b","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e44f865464af947a796090521cdb6730df4e505080ab9e71bd803d39788d7dc5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (High)","source_effort":null,"source_model_version":"grok-4.6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"22e499f8c169842e12f01e6b74be1b4e001c8a14e8cf930655b8b87fa03c5771","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.0,"score_display":"87.0","source_stated_rank":89},"run_fingerprint":"ce074935599c166c0537efe5830f8538363550ac0e00284516d4d70b5d07f14e","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"57c3387418ce295bc80043d54b64d8fb9e41aeb21afb44f1a57af10d463dd575","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (Medium)","source_effort":null,"source_model_version":"grok-4.6_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e24ca032c1ea1301ce1fe3b83d3b6d110f2adc23cc90e3a5328461dbe0356eb7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.5,"score_display":"87.5","source_stated_rank":84},"run_fingerprint":"d861ea46b9dc4f35b2741433e237365f691fc93a76647ee6c85b01b52f98dba0","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aa933f09ed572399447a968958ea3bc29727be253ced35d6333cdbd171f1becb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (Low)","source_effort":null,"source_model_version":"grok-4.6_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"4532c8b7706d4a7021ca15db7a685dd833a9e1ae3300afa2edcc7a669c7faf0a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.83333333333334,"score_display":"74.8","source_stated_rank":126},"run_fingerprint":"97a68e09fe89a93ef9d4107f81fe4f39dddabcb27b4b309f57618e31ccc98aa7","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"22b5623c2ff54322719b2d5e9e0fc523bbdd07034416626a189a4d01bb67766b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.784684,"date_is_proxy":true,"latency_seconds":604.494,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.918},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":null,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.6"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"97b8272ce6b7296d9b704937e3275f22ce9b7a7b42bab77370bbdaac27dbeda7","result":{"confidence_high":97.39927999999999,"confidence_low":93.80072,"sample_count":500,"score":95.6,"score_display":"95.6","source_stated_rank":4},"run_fingerprint":"6a7f6a8e39cadda975a00142dfff2a6f94ed50bc35f7935dde6a0662f39ddeb3","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"c7041244874e56c537e91fe46a135aa4d9048b753fbbdfaceb0d592e00e46787","metric_details":{"ci_attempted":451,"ci_half_points":2.175684224405049,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":301,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":87.21951219512195,"mean_cache_tokens":9079197.80044346,"mean_cost_usd":5.497668745011087,"mean_duration_seconds":1272.3478393858093,"mean_input_tokens":9344022.093126386,"mean_output_tokens":71403.54323725056,"median_agent_steps":81.0,"median_cost_usd":4.63937,"median_duration_seconds":1245.896303,"median_input_tokens":7667197.0,"median_output_tokens":71285.0,"median_output_tokens_to_pass":70062.0,"median_peak_context_tokens":156581.0,"n_attempted":451,"n_passed":301,"n_tasks_attempted":113,"n_tasks_passed_any":96,"pass_at_4_points":84.95575221238938,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_grok_4_6_xhigh","source_model_version":"grok-4-6","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"951d29baf7cfd97c4c65aa4eddbb9245d35b13cfb7b196883aa8dc2e169aa296","result":{"confidence_high":68.9162607210791,"confidence_low":64.564892272269,"sample_count":451,"score":66.74057649667405,"score_display":"66.7","source_stated_rank":24},"run_fingerprint":"8c32674f624aae890cd7972acb429e7445b7f4811a32e542dcd723e9e6ffa727","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"8ed04ca3986d738f3bb5dfd24c2ed5ac97bb71de246e9c47c2e0819bed5dcf1f","metric_details":{"ci_attempted":451,"ci_half_points":1.5336008186572951,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":294,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":78.960088691796,"mean_cache_tokens":7119379.299334812,"mean_cost_usd":4.3848732239467845,"mean_duration_seconds":1083.6365204833703,"mean_input_tokens":7348488.252771619,"mean_output_tokens":61160.94456762749,"median_agent_steps":72.0,"median_cost_usd":3.693604,"median_duration_seconds":1031.198421,"median_input_tokens":6003946.0,"median_output_tokens":60060.0,"median_output_tokens_to_pass":59255.5,"median_peak_context_tokens":136248.0,"n_attempted":451,"n_passed":294,"n_tasks_attempted":113,"n_tasks_passed_any":96,"pass_at_4_points":84.95575221238938,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_grok_4_6_high","source_model_version":"grok-4-6","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"6853fef79baf9a17fe0867d4949b777ef133e7b81457d2cd3fb6eb6e0a80207d","result":{"confidence_high":66.72207088517614,"confidence_low":63.65486924786155,"sample_count":451,"score":65.18847006651885,"score_display":"65.2","source_stated_rank":28},"run_fingerprint":"a150caa3ce815bcc72a32051797af0ee74db71e4a7e3cb9f49b8143853e2503c","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"aaf50d65743b060830277b99a0319116972c3e5bd863fcf21e34d994dc5cee31","metric_details":{"ci_attempted":452,"ci_half_points":2.28080457287792,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":305,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":70.28982300884955,"mean_cache_tokens":5533327.0088495575,"mean_cost_usd":3.4489837522123894,"mean_duration_seconds":895.5380579646018,"mean_input_tokens":5725195.139380531,"mean_output_tokens":49763.99778761062,"median_agent_steps":66.0,"median_cost_usd":2.945834,"median_duration_seconds":860.0352045,"median_input_tokens":4705124.0,"median_output_tokens":48585.5,"median_output_tokens_to_pass":48346.0,"median_peak_context_tokens":116622.0,"n_attempted":452,"n_passed":305,"n_tasks_attempted":113,"n_tasks_passed_any":95,"pass_at_4_points":84.070796460177,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_grok_4_6_medium","source_model_version":"grok-4-6","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"c097998a9518e61d0753b8096b8c8d8bf8d55c9b47b4678a0604bdcacc58af0a","result":{"confidence_high":69.7586806790726,"confidence_low":65.19707153331677,"sample_count":452,"score":67.47787610619469,"score_display":"67.5","source_stated_rank":20},"run_fingerprint":"d2290b265b7f3d2f9aa7d4d796af022f810399f4d28e2234383fb5b826604705","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"5ecf0367b6ce6083fe2b3194bb1006e59edc3e3aff6ce107e60b474c85a38b91","metric_details":{"ci_attempted":449,"ci_half_points":2.318651764429901,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":187,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":44.18708240534521,"mean_cache_tokens":1566819.4922048997,"mean_cost_usd":1.0423604231625836,"mean_duration_seconds":325.05958362360803,"mean_input_tokens":1646919.8017817372,"mean_output_tokens":16458.3429844098,"median_agent_steps":39.0,"median_cost_usd":0.742614,"median_duration_seconds":273.420881,"median_input_tokens":1089649.0,"median_output_tokens":14937.0,"median_output_tokens_to_pass":14781.0,"median_peak_context_tokens":45300.0,"n_attempted":449,"n_passed":187,"n_tasks_attempted":113,"n_tasks_passed_any":78,"pass_at_4_points":69.02654867256636,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_grok_4_6_low","source_model_version":"grok-4-6","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"07c9604f82072c0fca0563894e5332feede4adc8a6624bd3b8fd869e50e910c6","result":{"confidence_high":43.96675866866153,"confidence_low":39.32945513980172,"sample_count":449,"score":41.648106904231625,"score_display":"41.6","source_stated_rank":57},"run_fingerprint":"97d05a1079b04204df208c17b840ed07b8659507fc04845e5c95ce1b8cf6044f","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9ef79788fd821db5f9ff543e1177f39e0ec1c981aebf70ae5efa6604aad56306","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"87.21951219512195","mean_cost_usd":"5.497668745011087","mean_output_tokens":"71403.54323725056","model_release_date":"2026-08-12","notes":null,"pass_4":"0.8495575221238939","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"grok-4-6 (xhigh)","source_effort":"xhigh","source_model_version":"grok-4.6_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"99177cc21603c0dd51917fa4ea8e9fc0866d5ece336dde3a4a34d5cc1bb29c55","result":{"confidence_high":68.9162607210791,"confidence_low":64.564892272269,"sample_count":113,"score":66.74057649667405,"score_display":"66.7","source_stated_rank":24},"run_fingerprint":"6dd83174a5501e4b572fb8e701650eeddde564b0952f33e3342c90fa3a3537cb","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"72ffa6dcc0bf7e06e79437fb482ed3da48ef3f3d055011ac480ab978648285f9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"78.960088691796","mean_cost_usd":"4.3848732239467845","mean_output_tokens":"61160.94456762749","model_release_date":"2026-08-12","notes":null,"pass_4":"0.8495575221238939","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"grok-4-6 (high)","source_effort":"high","source_model_version":"grok-4.6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"43ddb64e653126665b800daa55c20a2ac0c1a98800a8b88305bbe1d7b49eacb0","result":{"confidence_high":66.72207088517614,"confidence_low":63.65486924786155,"sample_count":113,"score":65.18847006651885,"score_display":"65.2","source_stated_rank":28},"run_fingerprint":"ca088722959079b6583d13d00e2ace49f1798f730ce300d2326c7f868805e497","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3648e82ece94aa90f5793d23418b8dfdc1642f1675d5a6b87a2dd1a2a92ac566","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"70.28982300884955","mean_cost_usd":"3.4489837522123894","mean_output_tokens":"49763.99778761062","model_release_date":"2026-08-12","notes":null,"pass_4":"0.84070796460177","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"grok-4-6 (medium)","source_effort":"medium","source_model_version":"grok-4.6_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"a4dfd37b678fffcb4a5f461d5dbd35c14b598fb7682576860aafe526b560cb6c","result":{"confidence_high":69.75868067907261,"confidence_low":65.19707153331676,"sample_count":113,"score":67.47787610619469,"score_display":"67.5","source_stated_rank":20},"run_fingerprint":"fec49be6dbaff4fb8408485359a96dae56591501aa211c90aa3cb5bd1399dcf0","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b2c088e49a3138193ef689110de1781db63b47a37f763f99c1ffac485ed752d1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"44.18708240534521","mean_cost_usd":"1.0423604231625836","mean_output_tokens":"16458.3429844098","model_release_date":"2026-08-12","notes":null,"pass_4":"0.6902654867256637","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"grok-4-6 (low)","source_effort":"low","source_model_version":"grok-4.6_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"b955cc4c7978510908438d3ca4d810dd0755082d5e422279c04250ebf6c0dd9a","result":{"confidence_high":43.96675866866153,"confidence_low":39.32945513980172,"sample_count":113,"score":41.648106904231625,"score_display":"41.6","source_stated_rank":55},"run_fingerprint":"3823acb75bc3b68da362af89339ea4a69f6c8bc79b0e3233f62c3c2fb9dc37ff","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"607ee2a86635873fffe21883f73456c8e384b23f7d1e61cda5e7c808f8edd06d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"6.1","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"steps_per_task":"56","tokens_per_task":"49814","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 Extra High","source_effort":"Extra High","source_model_version":"grok-4.6_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"17d92884bc514eb8418311b1423ceb793e58e8b50e07e352d850ef25a7a634fe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.4,"score_display":"41.4","source_stated_rank":23},"run_fingerprint":"23c4cae37194afaf81c6375906aae68fbd5b11d9a7de8ca8aa0bd8daecbe4e25","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"65e583e536612ef48cfafa6c4a207912ccc1504ea63a51019ffec0b32adaf417","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"5.2","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"steps_per_task":"48","tokens_per_task":"41387","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 High","source_effort":"High","source_model_version":"grok-4.6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"c65bba63cee30240c068cd75726e0b93122fd411924a261ba888a7a7e0284343","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.4,"score_display":"40.4","source_stated_rank":26},"run_fingerprint":"3080b57965276db26bc73feeb25fb2cf0de0bac408a4d2a7f4af635f22fd70e3","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"58ecae33df0f07ac9cf479782ed7341199b933e9d93ac47ff5e284ec0101ddf3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"3.48","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"steps_per_task":"40","tokens_per_task":"24893","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 Medium","source_effort":"Medium","source_model_version":"grok-4.6_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"e8bdada9d69846a33add24a2824e9a29d854fa037738b373f2035e8caa331057","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":36.1,"score_display":"36.1","source_stated_rank":32},"run_fingerprint":"59c3c788bcef4037cad52e3b83047a74a570148af3847556383ab32789aefde8","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e60b4d577eac90c36b795a4d3e9347c523af6b9ca2979337795071866dc22d62","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.25","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"steps_per_task":"32","tokens_per_task":"16307","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 Low","source_effort":"Low","source_model_version":"grok-4.6_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"5ab0c5b3c3902fd90d3984f6d3dc41bd1f8baae985e24d26054d7090cfa8fc9f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.4,"score_display":"33.4","source_stated_rank":38},"run_fingerprint":"50ceefd509be94b00f6efe9865b67b253430a4f2605169cf4ede09996a8c3e78","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"25ac0aedaec817aead683252074f14e9b63ea78aad461bf5ccd40d5b20e784bd","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6","source_effort":"high","source_model_version":"grok-4.6_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"grok-build","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"5b70a8512e640283f3787f677a72cf593d4e73fc125f674ceb0c28906a2bdfeb","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":48.010000000000005,"score_display":"48.0","source_stated_rank":10},"run_fingerprint":"8623d06c679bfe37dc07f360c94042ebac8c2ca951208088b22d69858acb5dde","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3ec0cb540cc4847c8a09b534f0f4550b3be583a29bc7526cee85a20c8ac5af2e","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":8.183386,"date_is_proxy":true,"latency_seconds":14233.553,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.091},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":null,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"high","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.6"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"b0f337b5de495b565c1aaaefc126a6cb410343cf04a8fa2da556fdc1f0c8a7e3","result":{"confidence_high":51.70936,"confidence_low":43.51264,"sample_count":null,"score":47.611,"score_display":"47.6","source_stated_rank":26},"run_fingerprint":"780f2fed50d018eee4f7479a0c05cef6de80ef1f30960d59067d24dc9fb97e60","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2f7bfaefba24843922eccd3d5de16667bb1aef49f27159fb2aa6482b0dcbed18","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.039923,"date_is_proxy":true,"latency_seconds":118.4,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.941},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":null,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.6"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"09575027d4188fb4c8646b111fe01c8f517ae6f637bfe476fe4c4fe6322ea785","result":{"confidence_high":90.06836,"confidence_low":86.37964000000001,"sample_count":null,"score":88.224,"score_display":"88.2","source_stated_rank":7},"run_fingerprint":"a394a0b49a073bafc519e36fd250291fc53585acb1bf6b59de21b7b88ddffdaa","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0e875d3f00a1ef50a9416d5386c2fae6ad93b1025388081c3ce2dc38427de13f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (xhigh)","source_effort":null,"source_model_version":"grok-4.6_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"757521bc7b31223e7a2fb332d5b4a369ec8fbe9794522df2f629bb7567342fdf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.6203703703704,"score_display":"51.6","source_stated_rank":75},"run_fingerprint":"3fc1459d26988f4885ef4d0d4d733f2bc8e3d298df30f9b1f151dc2002230c35","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a5d5fdd7fbe9d7475eb8c55deab843e73c2e253b6b87cafa26bb1c13815ba541","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (high)","source_effort":null,"source_model_version":"grok-4.6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"5377cf594f81ed26be3cf04a8af5c373fe74b1e05cba3b70a5bb91d758c308dd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.4814814814815,"score_display":"56.5","source_stated_rank":28},"run_fingerprint":"18e9348a0e874bb19f0ab9aea816298e0a3618b6ea851c46cc65950eebc21886","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d36efda526886b0acbe0a865f6928014b582a45e05ef2a7685db1b89c57b706d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (medium)","source_effort":null,"source_model_version":"grok-4.6_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e84562c80221597973645e46bc7820ca107dd32ea137c2fea8916b1c1b793af5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.6296296296296,"score_display":"54.6","source_stated_rank":47},"run_fingerprint":"b01f2fcd67c3da0f198604846ddf01dd69e2cbb144683833028e326df920dd7a","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"06619916d0f3d477ed37ec52f2c92ec0b6760929d6253805587a17c0d686ae3a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6 (low)","source_effort":null,"source_model_version":"grok-4.6_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"d934c2e598bda6b6898de98615cf60bdc26e47325fcbe16a159db4d3d504d77e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.3796296296296,"score_display":"48.4","source_stated_rank":104},"run_fingerprint":"5a43073646fc520c5d08a9d536f8d16e8e4517c61aceae74c50421341d8d0132","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4e36b7d0bb5fe226905a773823cb9a28927b61d6d4b2b705e14fe656395518e7","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (xhigh) · scicode"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-xhigh","upstream_model_label":"Grok 4.6 (xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2dd36d488534dcbe4cca4631c2a9cde262cc039de2d6dad0e4b33d0ed5af95e9","result":{"confidence_high":58.695957,"confidence_low":47.243337,"sample_count":288,"score":53.009259,"score_display":"53.0","source_stated_rank":null},"run_fingerprint":"9d60c4828b99ba9a0076cb1d55aa223d2e59671f97e027967e57cb00f206b54b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d484e668be3236159ba614f1b6df28910c6eb95f19797006712dda64f006bb4f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (high) · scicode"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6","upstream_model_label":"Grok 4.6 (high)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"673016e8b1c9efe51e6dfff9de4c7d637b0c7407bcf17691ef3247e0f3a5230b","result":{"confidence_high":62.084973,"confidence_low":50.707354,"sample_count":288,"score":56.481481,"score_display":"56.5","source_stated_rank":null},"run_fingerprint":"dbae0a0f71e7a7ad5cbeaf5d9838aa8a1e8578e1a3d1f83336ef0e8d0cd62726","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (high) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"28a9b659191d40f06fada15040db3ec2652059bf96a206eba9127b550749e336","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (medium) · scicode"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-medium","upstream_model_label":"Grok 4.6 (medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b5863e10854a2299a421d78b95596b2f14a8bf6735e221ac9eb40661929371ff","result":{"confidence_high":61.522064,"confidence_low":50.128092,"sample_count":288,"score":55.902778,"score_display":"55.9","source_stated_rank":null},"run_fingerprint":"7c959cc9760fd46322217bad3fe7a1fad70fb7693d2211094d384b9084493e83","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1e6d3c4b700442e1fa891ce980ad0e846e7f2baf4e70656885a545fc1817a391","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (low) · scicode"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-low","upstream_model_label":"Grok 4.6 (low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"452ce5701808a042ee40fbe54586bcd12ad080c4fa82a6436843f82f6b3e86eb","result":{"confidence_high":55.165107,"confidence_low":43.69272,"sample_count":288,"score":49.421296,"score_display":"49.4","source_stated_rank":null},"run_fingerprint":"e46d2ab474c35349270bbb723e94fc3eb3d54d4a41d0778a81bafe9c1e08a82f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"548fc7dc167d65f499c9bf447a153cd09338d22be8cee0d87ea7c09b0b65b893","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.24005","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"grok-4.6-xhigh","source_effort":null,"source_model_version":"grok-4.6_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"c30da6b2ea8861545a89073da5255befc4b15d43db77b3f5292f4dcc43404be7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1508.08,"score_display":"1508.08","source_stated_rank":19},"run_fingerprint":"aa541d91e0f3eeefec91ca9a0cc1b9822b818cffc23e26881a0ac97c19c1cedf","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e9b9170e30c023b299b1c5710c40166f6bb0c1470e6e8913abaa279a9c83dae5","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":4.875513,"date_is_proxy":true,"latency_seconds":1530.145,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.824},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":null,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.6"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"b434a09747b27f48a4ca6ff7230f6ad2751870b3fae29a970fa630fa8b20d496","result":{"confidence_high":83.73404000000001,"confidence_low":68.74396,"sample_count":null,"score":76.239,"score_display":"76.2","source_stated_rank":29},"run_fingerprint":"e3b2cfc1c0311c5151534ff360ad56bd2163f238dadbc5ded86925b99a73c9c4","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8f0b37116a1f6be390738b571de939469e56c0b4ac905cd9edb046e2a751bc56","metric_details":{"license":"Proprietary","variance":15.996807182374953},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"grok-4.6-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2cf5d1d0bf4cb6f5e8199db7821ef8716036eebece043c52d4a5b10a2483f782","result":{"confidence_high":1627.3721751584333,"confidence_low":1611.6940278120562,"sample_count":8315,"score":1619.5331014852447,"score_display":"1620","source_stated_rank":23},"run_fingerprint":"fe787c3849815e54ec815f6f5f5aaf37441040bf8770d8d3bf5ee0605de321dd","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0b0479e67524bc57fe03bb07dd5e9535c4688bc099502d264e8ec6442bbe8c21","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4.6_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"f37490ed195d149180e4d393dac8cde6abdc601c6be59ce33f4c5f618514e576","result":{"confidence_high":1627.79,"confidence_low":1607.49,"sample_count":4282,"score":1617.64,"score_display":"1617.64","source_stated_rank":16},"run_fingerprint":"fe2bcc6182e5f9ce5b33577966bfe42ff12a83c5e6395d886aab050a8617fd68","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"79abf2701f6154f44b7a4d4809a66de3bfe7bbf6e0805456f9cea96e8bc7a328","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4.6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"97d8f516d27f31969b45f7d72b7d15b3004d5a73f2cbdb2d0d84de6d1740d7eb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.29,"score_display":"67.3","source_stated_rank":37},"run_fingerprint":"89dc57dc5e4efbfc7ae5acbc24a8b40ea4ffcab88c79ed6f2e5527b72e87c472","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"789e663907f7f02289147f4f60f71f54e2bc283e0c16b7d4e6e0e828a5061ae9","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.453642,"date_is_proxy":true,"latency_seconds":642.908,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.085},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":null,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.6"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"8aab93c97dd6864afd6248884239f5ecc43813b9663866e804d2b24d0e4a3011","result":{"confidence_high":82.3636,"confidence_low":74.1904,"sample_count":null,"score":78.277,"score_display":"78.3","source_stated_rank":13},"run_fingerprint":"fee4f838efaddda150c014c076779a3fd0fc2fe35675514ec5cf5020ef71f052","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4be2ca872679db08bf7fe16f7896ae2ff115762dc5c90eaff7217cb6fadcb26c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (xhigh) · terminalbenchV21"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-6-xhigh","upstream_model_label":"Grok 4.6 (xhigh)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"fa4c67e765dae8e68887bb15c266cfdee3ddaa17a7f6ecc51bd48eb120061c3c","result":{"confidence_high":93.233339,"confidence_low":79.650655,"sample_count":89,"score":88.014981,"score_display":"88.0","source_stated_rank":null},"run_fingerprint":"4efb144d99b2dd7382caa6da2915d4d4defdd718be83f0932dc256b67a11a548","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (xhigh) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4724259e1a5403b3781424e1f3ea67116f4e6e86ed79984f1ed0218d96c4ac65","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (high) · terminalbenchV21"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-6","upstream_model_label":"Grok 4.6 (high)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"cf0be7349250fd6af408492ac0e46c770e3e189406ad46b8c3ee525bf3a18232","result":{"confidence_high":93.508285,"confidence_low":80.093778,"sample_count":89,"score":88.389513,"score_display":"88.4","source_stated_rank":null},"run_fingerprint":"5320a4295becb7e9d7ce374167a628e6abb1e5533dd7c168fbb936c1c73ba735","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (high) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6de2b3140c740dc38b547eb9d91f6af22db1323d35dd2857af27cc005e324d1e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (medium) · terminalbenchV21"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-6-medium","upstream_model_label":"Grok 4.6 (medium)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"927373a36e6b1a717bb2df8d39666e8aa635b05ffc7b995a7ce25066b7d8f323","result":{"confidence_high":90.392278,"confidence_low":75.311028,"sample_count":89,"score":84.269663,"score_display":"84.3","source_stated_rank":null},"run_fingerprint":"5c25fa59a8ba7f4f63fedf1fe1d0f733ea9435d51caaeb5ce7c7b7f42cc9b715","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (medium) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"396f1afb27f62419f54fc2064b14f03b7d2c47e49864def915aa0bf36e4326f9","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (low) · terminalbenchV21"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-6-low","upstream_model_label":"Grok 4.6 (low)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"134ebd7ac30aa03ea554a55e488021e730de3179e073b7a4178e418e90b8bc64","result":{"confidence_high":83.071885,"confidence_low":65.397767,"sample_count":89,"score":75.280899,"score_display":"75.3","source_stated_rank":null},"run_fingerprint":"1278f5be3eb215dfce785023bb2f5e55068213f65bcde3e99151a86f81cd9aa2","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (low) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T22:31:18.697086+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T22:31:18.697086+00:00","status":"fresh"},"measurement_id":"ea737046f2ed875df0461c20cf6120ade4fcb00c85ef37916db65f211a2c46af","metric_details":{"accuracy_stderr":1.49,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:07:32.471449+00:00","total_cost_usd":2095.44,"total_tokens":2851507900},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"xAI","url":"https://x.ai"},"agent_url":"https://x.ai/build","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"61adb66e-1cf1-4194-b68d-703554d21193","model_release_date":"2026-08-12","model_url":"https://x.ai/news/grok-4-6","n_trials":370,"source_row_date":"2026-08-12"},"run_count":5,"scaffold":"Grok Build","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"cf3c5bbd00982510245a76df7e2e0a8eafa12c3cdc98a1b67f186bc10a9160d2","result":{"confidence_high":37.521798,"confidence_low":17.77871,"sample_count":74,"score":26.49,"score_display":"26.49","source_stated_rank":5},"run_fingerprint":"0d7ac4f6331e114e3c830017f0228b9791c6933cdef6d873db8a4695bd46724a","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5c85bc4f6fe9bb195b900bf4180f75ab1e9b9b24478dd16075e558063a532c52","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (xhigh) · tauBanking"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-xhigh","upstream_model_label":"Grok 4.6 (xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"849330db086932ad1604dfa534e630eecb6bd61c5a7d8bcded83bb69a578f5f9","result":{"confidence_high":53.228596,"confidence_low":33.879899,"sample_count":97,"score":43.298969,"score_display":"43.3","source_stated_rank":null},"run_fingerprint":"2ed5bdfa5ee7fdd13b36a56dd668e47205f0eb44844e4249353ed6c826d83403","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (xhigh) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f38b622922a4327abc669ec802806fa7a6334911b3a5814b44c6aecd21434412","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (high) · tauBanking"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6","upstream_model_label":"Grok 4.6 (high)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"de408eee175a7976807b049c07799b9cdd184a54e3c0674e71222bbdfa258cac","result":{"confidence_high":60.4522,"confidence_low":40.936115,"sample_count":97,"score":50.721649,"score_display":"50.7","source_stated_rank":null},"run_fingerprint":"4e74c3312289d12f8db41ea1efcfcdbc7bbbade76dcb808f9b127151728e57b9","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (high) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0e2b299dd106cfa0496fd219a2994fe90a48647facc9a9a570ea6ce415fe9f95","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (medium) · tauBanking"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-medium","upstream_model_label":"Grok 4.6 (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"e7a12f945381b4dad34bd08e5bd216bc587e5528ae6782a1a66323c8074b7efa","result":{"confidence_high":54.244374,"confidence_low":34.84743,"sample_count":97,"score":44.329897,"score_display":"44.3","source_stated_rank":null},"run_fingerprint":"900e80217250d6c6cf45c32365abd6866def0448b7def7c3f020934d60370b06","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (medium) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7f20131b5fa8ee8d69f189bc78599d0db3b82d33e0706442a8b284669a8e4895","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.6 (low) · tauBanking"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"grok-4-6-low","upstream_model_label":"Grok 4.6 (low)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"08a572266436a5bba7a9ce56cbf03d90b0988f42b3a4325aa38ecb38e3cbb359","result":{"confidence_high":48.08744,"confidence_low":29.104512,"sample_count":97,"score":38.14433,"score_display":"38.1","source_stated_rank":null},"run_fingerprint":"2225c65ad4bccbd35e17e12327c31adde9d5cdc8d3295f43fb331a7c1052d90b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (low) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d62d81b2cdc94283abb2c6fa4e38334a267a299f79aff247376b5398843dfdd5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.773","mean_standard_error":"4.1","model_release_date":"2026-08-12","notes":null,"standard_error_points":520.0,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6","source_effort":null,"source_model_version":"grok-4.6_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"2d5050f5fdb55f3ca2bb296f033c004b9d15f35ee78c0e3c6a0c2076cc5a49c9","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":65.3,"score_display":"65.3","source_stated_rank":6},"run_fingerprint":"360ccdf88239e18ca90b3f7ff2acf2cd4f87ed772c29d7263321fca99bf07522","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_42c5eb67e57e5181f3df3cfe","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"37d49b0eea933a949f009e92e2a5bcdbe84dcea401ddf17035a2040133895e8f","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":27380},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Grok 4.6 (xHigh)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8e6b2b5c1cc12bc9c9eba83d1aeb4332acc74f8ee05e033c45f2379d937a8d87","result":{"confidence_high":0.02282100108568126,"confidence_low":0.002784269876943515,"sample_count":3131725,"score":0.012802635481312387,"score_display":"0.0128","source_stated_rank":25},"run_fingerprint":"7777593d2dabb249e14adb3b052fcaafdcab7a2864e8094063e3eb7aba83960b","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"eb044c62560444faaec6c9a88b267aeb707c015f13561c744910f726623e315a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-12","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.6","source_effort":null,"source_model_version":"grok-4.6_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"a904ed276dd3469869303926a854441ebe2a2f6ee78e0ccb7665fa02faf182d4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":9047.032500000001,"score_display":"9047.03","source_stated_rank":7},"run_fingerprint":"465ed940e276cc07b81eee66a8955bc6e09ce7029ead6560d9734b13e632f989","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acb04e327788bc5d55cd1774","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"adca3df2e04dcc3ced505341226c4394549a2eed1ad554c6ba94a4b11a2f55b6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Grok 4.6 (Xhigh) · gdpval"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-6-xhigh","upstream_model_label":"Grok 4.6 (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"f04554a6ab4c9b43c635c8863372aa4139eb8fa7ff70f5487cac763bbc199f26","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1632.22,"score_display":"1632","source_stated_rank":null},"run_fingerprint":"4ded34d4694504418d11a8eaad66240b13031329c00942403d8ad75d76609c6c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_466c9201a70dbe424a06eaf9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"64efb1a86c5ddf0edf183ccef0df33613f6203a2577affd5da50d8bdccc6d3fb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","reported_confidence_interval":"-20 / +20","source_locator":"Leaderboard models table · Grok 4.6 (High) · gdpval"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-6","upstream_model_label":"Grok 4.6 (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"36d37291419857c8ab4d307c95ce82e19253d27e9004f4f0130e96119c3c9c17","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1609.04,"score_display":"1609","source_stated_rank":null},"run_fingerprint":"249d7f90786cee6818351d50fed0fc17e8232c58fc000f4d600b7ef542e28b82","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bf39f31a22c70a3b0ed8abd8","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"999d889bac3b3722a7bdccdb81375ef3e94adf67f6acedcc728277d3f5cf4262","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Grok 4.6 (Medium) · gdpval"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-6-medium","upstream_model_label":"Grok 4.6 (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"7ac1e173d242d2f0f8386e1f73a9aa967c24dae10a63a1862fad5313f4de41d7","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1604.84,"score_display":"1605","source_stated_rank":null},"run_fingerprint":"e465cabcce2db612265155ad370550bfe7a296a30229c8aaf6068de61fd6d5a7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_60db1b9ea8db9fc5db871102","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"26b6e6ea468d4e6798eeba80c08f0b41247c5d010bbdff13bf76157e71cf5c1c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-12","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Grok 4.6 (Low) · gdpval"},"model":{"id":"xai/grok-4-6","name":"Grok 4.6","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-6-low","upstream_model_label":"Grok 4.6 (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"7b03beb890ce816555359586f4b9c9f07c1ef749b7bd9c2c8d84fc69db058e71","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1402.42,"score_display":"1402","source_stated_rank":null},"run_fingerprint":"1079885ab9978156b773a1acaea4543741965d0182ae9d033de58067ee98b14b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.6 (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_97097e4c35b1796c2852e07e","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b0f3a8112aa87b285a3e586b1be69ad125b22716cdeed555f0cfe18deff6d1cb","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-08-02"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen 3.8 Max; model release: 2026-08-02","source_accessibility":"API access","source_model_name":"Qwen 3.8 Max","source_model_release_date":"2026-08-02","source_model_versions":["qwen3.8-max_unknown","qwen3.8-max_xhigh"],"source_reasoning_efforts":["xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"10629ba7d05e86488bbfb358ed7454334e34a0a542d438c6b22fe1eb6d9c9e9f","result":{"confidence_high":158.81,"confidence_low":154.54,"sample_count":null,"score":156.55,"score_display":"156.55","source_stated_rank":20},"run_fingerprint":"c6d9d0b7dee88d57f48bc1cdafc533d73c2859184db5a16735c91bb2e27362b6","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d877af62a921b2b206ad3009","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T08:30:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T08:30:22Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d13acc45db9e67a139334dc1edebebfc7329b5c18f63abbd2ac4d3687dd9a083","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 Max · intelligenceIndex"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"4599416d9ace6f0860acb409bf91519915aed1bcc20500f9d3d789ba5eeedd82","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.304862,"score_display":"40.3","source_stated_rank":null},"run_fingerprint":"66ed635ec23ba817a7f0e028e9a8c32927139ef88fdacae950ed6e6859232d1c","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6b0f9f89a407f75ea5e8b2a","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dad88cb01f08670c27d6a7936bfea76f64d6159e76457282b1e47633683bd7c9","metric_details":{"license":"Proprietary","variance":7.104249477209426},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1486.8410720456059,"confidence_low":1476.3929752201284,"sample_count":23353,"score":1481.6170236328671,"score_display":"1482","source_stated_rank":23},"run_fingerprint":"e1769d67de19918bfa92725f48bf85a33352f18b1bce717de229385adea65e78","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0d5bba5bc760b957bb971b9f","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"20b22192a0d3366b86a90d13f417a538dab2c3aadb47d703044058cf13bc4021","metric_details":{"category_scores":{"Agentic Coding":64.64666666666666,"Coding":72.872,"Data Analysis":78.413,"IF":74.08349999999999,"Language":79.68733333333334,"Mathematics":91.31225,"Reasoning":88.2115}},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.46089285714287,"score_display":"78.46","source_stated_rank":17},"run_fingerprint":"db0a9c1e2e0bfa4f3a1c6aa45d863106b62f8939aacd3350b8297bc9c23cee37","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b41ea21204b5a7e098c39301","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6b98bb1080651d22a9cf1223742e7aeb9368af1f6c2d588c629945a73931649a","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 Max · hle"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"16c4d8cdbec0efd3e09029bc345c875326c99fb0fe02090c1b765e95eb3285d4","result":{"confidence_high":45.148768,"confidence_low":40.974175,"sample_count":2158,"score":43.04912,"score_display":"43.0","source_stated_rank":null},"run_fingerprint":"e2cc90045c78df263bcffda571d90a352f9f17b52cc0bcc99805e13e74407f84","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6b0f9f89a407f75ea5e8b2a","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:30:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:30:55.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1825facdca3d03bdf61cdc8618c7598afc4b98100f6eec2c598dc8416fd2c0ea","metric_details":{"best_score_across_scorers":"0.458","model_release_date":"2026-08-02","stderr":1.5763390640483554},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bfftd5EzUK9yAekafEofvK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fbfftd5EzUK9yAekafEofvK.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bfftd5EzUK9yAekafEofvK.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"3f9353637d7d4fefaf798304b12dacc4c53ba0e42ea81f1284993764f419ef4d","result":{"confidence_high":48.88962456553478,"confidence_low":42.71037543446523,"sample_count":1000,"score":45.800000000000004,"score_display":"45.8","source_stated_rank":37},"run_fingerprint":"9dc1badf4f166c6a1e9e6e7a22f8d368776cdd919b16c0e29fc59f95980aaabc","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9f288de94feeced4427eb208","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8fa6924790b2be2096d7f1a0c38e2f2a2528b6fc030481850f1513a3fe087d7d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.077028,"date_is_proxy":true,"latency_seconds":266.606,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.221},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":92.424,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.8-max","zero_shot_accuracy":94.949},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"bc1edd283d0596cd623bc077d95d7a1cc253ddc0695af6c5e8cd776c0198248b","result":{"confidence_high":96.07916,"confidence_low":91.29284000000001,"sample_count":396,"score":93.686,"score_display":"93.69","source_stated_rank":6},"run_fingerprint":"b428407cb138f2f8399dc61c0da380e3664f10d78ec73f823f8bb45ac0ab3801","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d8aafdf161c3cb55a509389e","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"760d3df8000f58c14109756e7264ea46e466c20a263413934870f14d240c0f50","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 Max · gpqa"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9e6fe41bba22b39b93cdefceec89b41c62758c665846d9207e7d2ef0cb5c909d","result":{"confidence_high":95.587831,"confidence_low":88.24028,"sample_count":198,"score":92.727273,"score_display":"92.7","source_stated_rank":null},"run_fingerprint":"f81d35300982f47c0adc100b84a3872aa674452261873b47828cf1b8d21a8d4f","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6b0f9f89a407f75ea5e8b2a","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":61,"at":"2026-08-04T16:28:46.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-04T16:28:46.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3d64d933439472be88bf7e43358cc33155b623c61a857cca532c99966c0dd13b","metric_details":{"best_score_across_scorers":"0.9267676767676768","model_release_date":"2026-08-02","stderr":1.686498021450146},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"TEVNDC9QXBh7PQvX9kHZip","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"eb33b583c7b6e965d2266a8abb7975d764aa2a3787787e1ea1dbbcb5588eee45","result":{"confidence_high":95.98230379880997,"confidence_low":89.3712315547254,"sample_count":null,"score":92.67676767676768,"score_display":"92.7","source_stated_rank":24},"run_fingerprint":"8a9e6aef335c64e408e62db3b17b67845ee897556cca71da0ac4eb643a302568","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9f288de94feeced4427eb208","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bcdb8962ff13aafb852d318c7581e3f0e4989769f53742ff463852cef81f5ab6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.018585,"date_is_proxy":true,"latency_seconds":63.576,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.313},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.8-max"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"be54c8ea1f980e799499f5645eb3d392487571e73a3fce3fd5f014663236e450","result":{"confidence_high":89.21548,"confidence_low":87.98852000000001,"sample_count":null,"score":88.602,"score_display":"88.6","source_stated_rank":18},"run_fingerprint":"473275aff49082eb0b0c9c7ce69fffc6d4cd2b3ccd8930836b386c7cb00718c7","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c9dab4aa7fa2f362d4a64f77","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":61,"at":"2026-08-04T16:28:46.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-04T16:28:46.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4b93765bd8b8f1419cd9a7cc7e8c1078414d4093e5d23cc98d8c764d6cd9839a","metric_details":{"best_score_across_scorers":"0.9944444444444445","model_release_date":"2026-08-02","stderr":0.3883473863441843},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"aGRKHaq9uSawfYVpC565zU","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e63e8660218051fe73ec5b9f865c4b6e4182286cf8945c04ba7347e139856fc4","result":{"confidence_high":100.0,"confidence_low":98.68328356720984,"sample_count":45,"score":99.44444444444444,"score_display":"99.4","source_stated_rank":14},"run_fingerprint":"cc8b341dad7e91da80bb5ce30adb9a6da5943d92fca07e58fc578e1d1fdb6f02","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9f288de94feeced4427eb208","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":61,"at":"2026-08-04T19:32:51.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-04T19:32:51.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"87c2c554913e0cfa04649cbd46ec5b3ffa5d99f60aa0847c1a5312cd9abd44a1","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.7473684210526316","leaderboard_private_problem_count":285,"model_release_date":"2026-08-02","public_example_count":10,"stderr":2.57841025556124},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"ABT89ieSWGnGj4RYMduHiG","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FABT89ieSWGnGj4RYMduHiG.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ABT89ieSWGnGj4RYMduHiG.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.8 Max (xhigh)","thinking":null,"upstream_model_id":"qwen3.8-max_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"7c3324d48b24f2962ccf8492e8bb7abd941cae2adad1c1b4cda5aac279ed9ba5","result":{"confidence_high":79.79052620616318,"confidence_low":69.68315800436312,"sample_count":285,"score":74.73684210526315,"score_display":"74.7","source_stated_rank":18},"run_fingerprint":"c8ead37fa47bd35318a6185c0d076f8bd18edcc09f883a7b615fe63fa1f92df6","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9f288de94feeced4427eb208","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":61,"at":"2026-08-04T19:32:51.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-04T19:32:51.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d499382974531c984ae4f074c3d9cde5c81570485dbcbdc0276e6339891cc485","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.4634146341463415","leaderboard_private_problem_count":41,"model_release_date":"2026-08-02","public_example_count":2,"stderr":7.884502378168105},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"PBRdMZyz4L4CqPpXyLD2Lk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPBRdMZyz4L4CqPpXyLD2Lk.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PBRdMZyz4L4CqPpXyLD2Lk.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.8 Max (xhigh)","thinking":null,"upstream_model_id":"qwen3.8-max_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"994602caa30af06c383b9d649b7a2039606845c9ba9f3093dd42ace2159cb112","result":{"confidence_high":61.79508807584364,"confidence_low":30.88783875342466,"sample_count":41,"score":46.34146341463415,"score_display":"46.3","source_stated_rank":23},"run_fingerprint":"906b569e8e1e867a32df1fa76f8834d7499890bd20d5be1af8122a33c35ed175","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9f288de94feeced4427eb208","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"02c73d092266eda982804681c70d76a91b5dfe03de3c943e93953df4e72cc7d1","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.015881,"date_is_proxy":true,"latency_seconds":43.853,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.78},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.8-max"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"75624c4bd3e57101af0cc99d4da5792ebde501d5659006e3cd38be3b338c3257","result":{"confidence_high":89.5638,"confidence_low":86.50619999999999,"sample_count":null,"score":88.035,"score_display":"88.0","source_stated_rank":12},"run_fingerprint":"8f6befa781b48547bcf27ff0238a3f7c546d13e6b1d4c4c489df5be824a11acc","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1deffe7efba519db4b67e6e4","provider_config":{"source_id":"vals-mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d2eca5b033ccbc573f6d1e63e6773c48d52911d0dda1d5453fd940e57e59152b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 Max · mmmuPro"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"953cd55bae372b6122e6f72952195d8678adf864620e68d896b60f346c957c5f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.312139,"score_display":"82.3","source_stated_rank":null},"run_fingerprint":"ac0db3b5089e9a6f463b695acc58ef7eaf12e048d2be0d3bfb3c944f4fff3817","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6b0f9f89a407f75ea5e8b2a","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f03bbf7cca056d965d18721168581d175d6a699549ebf6a56da218cd5e2a21cc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-02","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 Max","source_effort":null,"source_model_version":"qwen3.8-max_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8ae55df023be0e7f2e62d5823972ffd02ac3245bba85d29d76faeaedb4b39410","result":{"confidence_high":30.732218,"confidence_low":12.347566,"sample_count":71,"score":20.0,"score_display":"20.0","source_stated_rank":53},"run_fingerprint":"ec5cf53db0f98bf1b6eac51eb565bf3d27818b0981018ffe476bbdc20d6c06f8","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1718ae78cff960c44c0c9653","provider_config":{"source_label":"max"},"provider_mode_key":"Max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0c7e4d670d9afb1039572a87167591cc247bad3f08ece2cd99349aaa07245697","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 Max · critpt"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"92db85f82096eb09819b2939be4890a2624603c35d15b3ac8d560978ed415215","result":{"confidence_high":30.816865,"confidence_low":12.304628,"sample_count":70,"score":20.0,"score_display":"20.0","source_stated_rank":null},"run_fingerprint":"895fc3ae11292a369946a9cc64c78bac6839e98fb342f18618779f29e4e099e3","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6b0f9f89a407f75ea5e8b2a","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":61,"at":"2026-08-04T16:28:46.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-04T16:28:46.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fc1363cbbff3287d36ad69d129aadd4dde41f22c6ff798506854ea5d1171edc3","metric_details":{"best_score_across_scorers":"0.29","model_release_date":"2026-08-02","stderr":4.560480215720686},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"PPbjbUncV6qa9jNRKjYrfp","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"61afbe4e8d6e213b93a8c9deb24aabc1462824d4f23f69d06ee7e2e1e29aa55f","result":{"confidence_high":37.93854122281254,"confidence_low":20.061458777187454,"sample_count":100,"score":28.999999999999996,"score_display":"29.0","source_stated_rank":54},"run_fingerprint":"31036f90cd442b28b90b7d476cba01cfe8defc81b1dd8eacfa5274f309fbb8d3","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9f288de94feeced4427eb208","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2e2dad5f8e943197b00b396d042e0197bbbb135139d42b1434433fbd481c507d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.128442,"date_is_proxy":true,"latency_seconds":2502.097,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.572},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.8-max"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"5fc2b366fbbd5a0a5534b485ced79936c3717e4db026f45ce18d7bbdbfaa77de","result":{"confidence_high":88.68111999999999,"confidence_low":82.51888,"sample_count":500,"score":85.6,"score_display":"85.6","source_stated_rank":17},"run_fingerprint":"7803aa5e91dc090c16f7c0bfccde0ea8d01a2fc034ddff45eb6bcebc0466ded3","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5f68ba2f45063ad59689c8b8","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"dbfb894347faec92ba083ffdca70d7b1d28a2e973311086b346022a40449d2f7","metric_details":{"ci_attempted":449,"ci_half_points":2.6627926725075453,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":258,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":111.33853006681514,"mean_cache_tokens":12477251.240534522,"mean_cost_usd":3.7290646500445437,"mean_duration_seconds":2567.2746634654786,"mean_input_tokens":12996066.461024499,"mean_output_tokens":95075.18262806236,"median_agent_steps":102.0,"median_cost_usd":3.132689060000001,"median_duration_seconds":2424.198128,"median_input_tokens":10163801.0,"median_output_tokens":89729.0,"median_output_tokens_to_pass":91083.0,"median_peak_context_tokens":160572.0,"n_attempted":449,"n_passed":258,"n_tasks_attempted":113,"n_tasks_passed_any":94,"pass_at_4_points":83.1858407079646,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_qwen3_8_max_xhigh","source_model_version":"qwen3-8-max","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"a7141b72f5abab97686747a8f456230d9df3fc1dfbefe0093c97e1efe26855df","result":{"confidence_high":60.12381717139396,"confidence_low":54.798231826378874,"sample_count":449,"score":57.46102449888642,"score_display":"57.5","source_stated_rank":37},"run_fingerprint":"9027b13ee5edc2c1d2b927df6d7cc4e6fd47c4cc4f9354b3322c513c60c50612","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9f288de94feeced4427eb208","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"98a5bc18ecbaa67605a61be910fb4cce6bcf3223ece38322db23167b7d2578b3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"111.33853006681514","mean_cost_usd":"3.7290646500445437","mean_output_tokens":"95075.18262806236","model_release_date":"2026-08-02","notes":null,"pass_4":"0.831858407079646","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"qwen3-8-max (xhigh)","source_effort":"xhigh","source_model_version":"qwen3.8-max_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"74d5cf62fcd62801b25f8921498883deed1db9f04d7bd4f16fb429e3042de0ad","result":{"confidence_high":60.12381717139396,"confidence_low":54.798231826378874,"sample_count":113,"score":57.46102449888642,"score_display":"57.5","source_stated_rank":36},"run_fingerprint":"eda41f5e3207358695c9ffb73f199a122ddec5904d65753d95c90c3e6cacf433","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9f288de94feeced4427eb208","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_0f94a5c34a87744f357d4816","split_or_window":"qwen-vendor-comparison","unit":"percent","version":"DeepSWE 1.1 · Qwen comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":54,"at":"2026-08-12","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-12","status":"fresh"},"measurement_id":"3ad13c0b31d9966fcad9bbebfc35bedeef9b1eba70fde19741983cfab3741fb0","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Coding Agent table; DeepSWE 1.1; Qwen3.8-Max column; methodology note 4"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on DeepSWE 1.1; highest score across Claude Code and mini-SWE-agent; temperature 1.0; top_p 0.95; 256K context; reasoning effort not otherwise disclosed","reasoning_effort":"reported","result_published_at":"2026-08-12","source_content_hash_method":"raw_bytes","source_content_sha256":"7690c680c2d6eb284179ef5dd4db63260d248531cae9087b09b2c2200230f331","source_published_at":"2026-08-12","tools":"Claude Code and mini-SWE-agent tool environments"},"run_count":null,"scaffold":"best of Claude Code and mini-SWE-agent","tools_allowed":"Claude Code and mini-SWE-agent tool environments"},"protocol_fingerprint":"26f7dbbbd59c10993e81bb31ec607ed81e91dce338e64d5c3ca6702cda935db1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.6,"score_display":"56.6","source_stated_rank":null},"run_fingerprint":"dc66eee95626e3125b2a59e3568ee801e3da6ae839a4847f1806017422c235d3","source":{"content_hash":"60e41c5cda83e653de6d55a51d96311dc9f5cca01af54cbccb44db4fcdeb4295","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","id":"qwen3-8-a95b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen3.8-Max model card; metadata facts only","locator":"Coding Agent table; DeepSWE 1.1; Qwen3.8-Max column; methodology note 4","name":"Qwen3.8-2.4T-A95B official revision-pinned model card","redistribution_policy":"metadata-only-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_646bb68a28e877c1466e9d75","provider_config":{"source_id":"qwen3-8-a95b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"metadata-only-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_232b90e5b13ff602749b9048","split_or_window":"qwen-refined-vendor-comparison","unit":"percent","version":"SWE-bench Pro · Qwen refined comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":54,"at":"2026-08-12","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-12","status":"fresh"},"measurement_id":"58bfc222aa2a065be8805f7c4495aa3a44549d567fcb4c98fb74308bdd6806ad","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Coding Agent table; SWE-bench Pro; Qwen3.8-Max column; methodology note 3"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on the refined SWE-bench Pro task set; Claude Code; temperature 1.0; top_p 0.95; 256K context; task count and reasoning effort not otherwise disclosed","reasoning_effort":"reported","result_published_at":"2026-08-12","source_content_hash_method":"raw_bytes","source_content_sha256":"7690c680c2d6eb284179ef5dd4db63260d248531cae9087b09b2c2200230f331","source_published_at":"2026-08-12","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"ca23444044809f8156938debc846e7b2fff27eb273a37bfb51447632d47704b2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.7,"score_display":"67.7","source_stated_rank":null},"run_fingerprint":"288402006064b77ad7938e319328dc17bd7d06eadcebbf441047a167b61d7928","source":{"content_hash":"60e41c5cda83e653de6d55a51d96311dc9f5cca01af54cbccb44db4fcdeb4295","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","id":"qwen3-8-a95b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen3.8-Max model card; metadata facts only","locator":"Coding Agent table; SWE-bench Pro; Qwen3.8-Max column; methodology note 3","name":"Qwen3.8-2.4T-A95B official revision-pinned model card","redistribution_policy":"metadata-only-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_646bb68a28e877c1466e9d75","provider_config":{"source_id":"qwen3-8-a95b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"metadata-only-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2ad141d1f4c28759d2de28c07ec273dd4dd287ef97dc60888287cda0d59b1d22","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":9.468852,"date_is_proxy":true,"latency_seconds":6398.213,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.204},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":null,"source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.8-max"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"968d99861a49e871aa7039971861a6f59327b60ef12c5d4a09afb681dd01b3ec","result":{"confidence_high":79.08883999999999,"confidence_low":58.689159999999994,"sample_count":null,"score":68.889,"score_display":"68.9","source_stated_rank":11},"run_fingerprint":"e01b8bf2310d7a63ade63512b118f9fcb02b075721227a5634a8ae51b8e04a20","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dfe941467107f74de26697a2","provider_config":{"source_id":"vals-ioi","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dddb60df348075858222ab9524b0322b91edb766a92842152ca0d52eef3d476e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.091412,"date_is_proxy":true,"latency_seconds":337.788,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.951},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.8-max"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"b70365bbcd0591a13b2195857355723ec0eb4b0909fe9ab427a36b68e0cf2370","result":{"confidence_high":89.71596000000001,"confidence_low":85.98804,"sample_count":null,"score":87.852,"score_display":"87.9","source_stated_rank":10},"run_fingerprint":"b9ce39b34a79bd3004bf1c7dc364467c0ab187059e9a1b6f8a62222865c2a751","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a132c45f37998747339c62c9","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9483de8a04e94885ee0888dcd523ce7f7ce84f88bd82b29a69c18127533c85d7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-02","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 Max","source_effort":null,"source_model_version":"qwen3.8-max_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0d5e336fff5062ee74898454aa483a4105a5c3bd40078017e0f55fcaafbf6fe2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.8935185185185,"score_display":"52.9","source_stated_rank":69},"run_fingerprint":"e2f3253d65f68a3d038d0a8dca8ea0020e2be39de513132452f08387ecd0fd10","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1718ae78cff960c44c0c9653","provider_config":{"source_label":"max"},"provider_mode_key":"Max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"86bef2cfc74a2b1c4e96922cb76336442c5637de5b271816e28a4ef2d723959a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 Max · scicode"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"885e7e5e3b90cc56409e84d866138bbf8f8af55af0bbed7a76423f6a6d10c331","result":{"confidence_high":58.922751,"confidence_low":47.473413,"sample_count":288,"score":53.240741,"score_display":"53.2","source_stated_rank":null},"run_fingerprint":"da378750003762a2c1e3f3ad7afab922d2e29325609de8be4f545fad67ca1b15","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6b0f9f89a407f75ea5e8b2a","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"256f6a2ce448aab3aa001063a847c58d116174f5c6dcdb9705b100141676cf44","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":8.238852,"date_is_proxy":true,"latency_seconds":10135.139,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.362},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.8-max"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"adcf9383cb3a74b995d0d49ee55d0d11b9cbb6970f7f8b8dd185871d3df9914b","result":{"confidence_high":75.20551999999999,"confidence_low":54.186479999999996,"sample_count":null,"score":64.696,"score_display":"64.7","source_stated_rank":41},"run_fingerprint":"92e67f31b0a1e985068980a503a3038de7634dbfc375ca7e4925071e937bf2e2","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2fef4e63c947b0e5bf3ecf53","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"577c4b9ba8d973bb4b54eaaebbd742944e6a0e579ae5a42b195e44fd1b80014c","metric_details":{"license":"Proprietary","variance":37.76728768617468},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.8-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4e51a5abba1e77b3a444d1e9c1d8c4ecaf256b9e78eeb7fad06a4d7c60c3d565","result":{"confidence_high":1683.2003241618079,"confidence_low":1659.1103693743246,"sample_count":3454,"score":1671.1553467680662,"score_display":"1671","source_stated_rank":10},"run_fingerprint":"5102295881c4f140e16b0c79621efc2dea9da52cae6c1214e817097cee7bd739","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a6cfdc05f4f1b956853cc974","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6dc90c0f7bfac95558210dd8ef0b8ca2ce253c9d8149c7ef538a7083170dbeca","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-02","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.8-max_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"768d951df0a9c4b96d961edd7a8859b0c0081e0a7a315e9929c31002bfa942c1","result":{"confidence_high":1683.04,"confidence_low":1658.11,"sample_count":3221,"score":1670.58,"score_display":"1670.58","source_stated_rank":10},"run_fingerprint":"978c25d3aa794bd51d62edffc0f4c082d9148fe0bd02b0d26fc94726b146d8d0","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a8c711085b6a9d3283f67a65","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cacadf3e9a481f30a124295e1c93a3c409992b2b5c0c635e5fc548ac7fdd3b5c","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.448084,"date_is_proxy":true,"latency_seconds":890.917,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.649},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.8-max"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"f045625ec89078ccd515acb6fe03a698a5c35c1faf96769361d33bb9ef0d19b9","result":{"confidence_high":68.68804,"confidence_low":66.14395999999999,"sample_count":null,"score":67.416,"score_display":"67.4","source_stated_rank":35},"run_fingerprint":"32d706345fdda53bc2ba502d5691d6eb1e34d418320b03fc04881e362759681a","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dad7b63bac0ecf821f75dc11","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5a3ccbade8645a3ceb8f8ffa5c719e9987f4723f1ff609816b442a039d527346","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 Max · terminalbenchV21"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"0aefec21a95bd2308494a41875730c0a84f369818a8394ce9b6fc27c67830f57","result":{"confidence_high":88.019946,"confidence_low":71.938807,"sample_count":89,"score":81.273408,"score_display":"81.3","source_stated_rank":null},"run_fingerprint":"8a4efef202d8e13788e3f16d0fec1f2e2a65d8788388d8b1ce0926b0112c1da7","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6b0f9f89a407f75ea5e8b2a","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":63,"at":"2026-08-03","basis":"evaluation_at","evaluated_at":"2026-08-03","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"741c66a454befb30de7a0b83b5f45c878f8bc61eebd097efa4614f4856ec5167","metric_details":{"cost":null,"pass_2":45.18900343642611,"pass_3":39.69072164948454,"pass_4":35.051546391752574},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"qwen3-8-max_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"ed98c8443e3b747933fc9595ca5529c72ec372d970230b74353a8645611b5d55","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.154639175257735,"score_display":"55.2","source_stated_rank":null},"run_fingerprint":"40145f1e028e4b5d57e72df8abb98e19589134f6e878381a0cf6106f6c802eb0","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_9f288de94feeced4427eb208","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a2cae52cb6b970c00d6e79dcaa09e9d96fb1b40de4484255101be888bcff3f18","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-03","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 Max · tauBanking"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"29ee69f49dde555d2727d00f373c0bd81259e7c9bd82007665bd401de1d6542e","result":{"confidence_high":61.044798,"confidence_low":41.533503,"sample_count":97,"score":51.340206,"score_display":"51.3","source_stated_rank":null},"run_fingerprint":"74f3788b22416d2ee905cc0d46c5e3b6b9ec73fff2b2a9726d7c9971c0e5ce18","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6b0f9f89a407f75ea5e8b2a","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_ff4a6e288c3c17d588478b46","split_or_window":"qwen-vendor-comparison","unit":"percent","version":"SkillsBench · Qwen comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":54,"at":"2026-08-12","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-12","status":"fresh"},"measurement_id":"5849cc4cc83c700fb7dfd173959c53d369bd6504526768d924bcb8a67384cb30","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"General Agent table; SkillsBench; Qwen3.8-Max column"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card comparison on SkillsBench; task subset, tool policy, run count and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-12","source_content_hash_method":"raw_bytes","source_content_sha256":"7690c680c2d6eb284179ef5dd4db63260d248531cae9087b09b2c2200230f331","source_published_at":"2026-08-12","tools":"not reported"},"run_count":null,"scaffold":"not reported","tools_allowed":"not reported"},"protocol_fingerprint":"5a87a249627548ba41ab1e22adf8f465931b323f1bc4cc1e11b5f7988d5b1a3c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.2,"score_display":"70.2","source_stated_rank":null},"run_fingerprint":"a2c923d59a904bd16ce46fb77f8585d4df9d2b251f29f53be9cc730557b5847c","source":{"content_hash":"60e41c5cda83e653de6d55a51d96311dc9f5cca01af54cbccb44db4fcdeb4295","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","id":"qwen3-8-a95b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen3.8-Max model card; metadata facts only","locator":"General Agent table; SkillsBench; Qwen3.8-Max column","name":"Qwen3.8-2.4T-A95B official revision-pinned model card","redistribution_policy":"metadata-only-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_646bb68a28e877c1466e9d75","provider_config":{"source_id":"qwen3-8-a95b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"metadata-only-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"88a44c212867a778b87817678e09555eb61d0dc0f939d15a300461e884dd23df","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":52824},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Qwen3.8 Max","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fee68b4b43e05251720050616a23b550e8fa137720ba56195c9e3dddf07bce19","result":{"confidence_high":0.03135111092092201,"confidence_low":0.01887951677229076,"sample_count":4577924,"score":0.025115313846606382,"score_display":"0.0251","source_stated_rank":22},"run_fingerprint":"33480d7750a320b09f307cd2ce1d6b8987b5a4eebdc161682ba10a57e4bc00a9","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b609cc1047aa681b2607f1ec","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"07eff92aeab845e5b58674194963a20f80ccae02087055500058b6cd0aa34d2a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-03","reported_confidence_interval":"-19 / +19","source_locator":"Leaderboard models table · Qwen3.8 Max · gdpval"},"model":{"id":"alibaba/qwen3-8-max","name":"Qwen 3.8 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-8-max-0803","upstream_model_label":"Qwen3.8 Max"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c331261352b9e462d4178d18ead210e23ebf4b52da3beee16537f95ebe6c2cf4","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1607.59,"score_display":"1608","source_stated_rank":null},"run_fingerprint":"77cae01dc45a342afa240e5fa5ab9f97319bfaa6b4d3eb8856e5a96160ae4482","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6b0f9f89a407f75ea5e8b2a","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cdd42f4ffee3214fdd82ab2647a59e36f45ff90f81571016bec72abdab446266","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-09"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.6 Luna; model release: 2026-07-09","source_accessibility":"API access","source_model_name":"GPT-5.6 Luna","source_model_release_date":"2026-07-09","source_model_versions":["gpt-5.6-luna_high","gpt-5.6-luna_max","gpt-5.6-luna_medium","gpt-5.6-luna_none","gpt-5.6-luna_unknown"],"source_reasoning_efforts":["off","medium","high","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"91120eea47a966dd4a64366fce098c20f61bb47f8d1d54aced2f39e2967bd1f9","result":{"confidence_high":158.6,"confidence_low":154.13,"sample_count":null,"score":156.45,"score_display":"156.45","source_stated_rank":21},"run_fingerprint":"0f72911db2ffecfa3ba237cfb5d5aef2c92de7b112cfb1449748f6c0375d94ad","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a3d98e32adc7050a02790a04","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"898b63078ca053bb25db65ed2542f511c56d02ec609ad19bba0ed7b6f12f2a1b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (max) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna","upstream_model_label":"GPT-5.6 Luna (max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"6f0dd2c499c0efdcd0dbe497b88dcd1c2d499077d87a876349c60ac90a40c08e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.504849,"score_display":"37.5","source_stated_rank":null},"run_fingerprint":"62b2acdaf4ef5cedbff48d3a68791695b35e4855c45b21fff792cd3e68446798","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"73adc645145e8368cb2057cfa8cf5978319504c5939081edd9e00cca241edc9a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-xhigh","upstream_model_label":"GPT-5.6 Luna (xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"006a4696457fd744f66bb32e84308a0db0aa4a18584ed0ce190c79228d3651d3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.772049,"score_display":"34.8","source_stated_rank":null},"run_fingerprint":"11a4d964525052f6c0cb1c5ea24d5ddd8687fe77e8471de33a1bb1dc8aa458ae","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":25,"at":"2026-09-10T06:23:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-10T06:23:57Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fce9eb1f62bdc7d85bc3bdcdcfe6be02d84bfc81d08fbca4967cf2b6591ee2ab","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (high) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-high","upstream_model_label":"GPT-5.6 Luna (high)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"5f5a15c784e407338c991c289cfb40f5df52d11a42b3673ade3611e46d8e4657","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.414795,"score_display":"32.4","source_stated_rank":null},"run_fingerprint":"dbbba80034e1ef289ddf73ea4f64f8a2cb8af1b8647d6d48393351c278614064","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (high) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":23,"at":"2026-09-11T17:49:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-11T17:49:02Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f0acdd49e5fd3169dab5e0b26fa6020ce6082373a24d5cd5c87218c996673350","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (medium) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-medium","upstream_model_label":"GPT-5.6 Luna (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"bd717766a69a89a5de195b094693f27e0f6063be46abc082746637ab5af31a63","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.477399,"score_display":"25.5","source_stated_rank":null},"run_fingerprint":"0060308f6a3dc3dd1ad880d68e265905f7d13fcbcab1af4e00cacf25f147e2ae","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":23,"at":"2026-09-11T17:49:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-11T17:49:02Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"64472717cab4630708a1916c4ec37bfc07413e6e0dab393f04d25d94129341c7","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (low) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-low","upstream_model_label":"GPT-5.6 Luna (low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"3d5cc5d68556f53ad8cce793b41f645ec3a1fca646e6e8bacff2060061dfde1b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":21.545603,"score_display":"21.5","source_stated_rank":null},"run_fingerprint":"00141c0f8a1dc9936855024edc382e3e114f0ee8076ac862e353832ad9179bdf","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"463580f0518c90405b5c8b163f09143352c25229451fd34a96c7904857a7c8e6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · intelligenceIndex"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-non-reasoning","upstream_model_label":"GPT-5.6 Luna (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"c795757e19e5f561e9e9c5ffa435f25933db40df78c10af1fcb249bd89677443","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":16.050025,"score_display":"16.1","source_stated_rank":null},"run_fingerprint":"34e2cd644021d8fd9677f87fbd2aabf13bee1917924674fa41aa2a4b7d52af67","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d021a16b4a611d4591e31aa59811a567ef44142761094e10fe22939ed2b1119b","metric_details":{"license":"Proprietary","variance":5.039075836622419},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1457.4499591369872,"confidence_low":1448.6505495384426,"sample_count":38275,"score":1453.050254337715,"score_display":"1453","source_stated_rank":75},"run_fingerprint":"9491a8de0660ccf8d6ce8e5a10d21fdb01ba04f0479db759f8fe0bbfc0f13cf6","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3dd5e42384f1f0d5790b3aa160d5490faef217b9dd251137f192578a1cde3bc0","metric_details":{"category_scores":{"Agentic Coding":48.434333333333335,"Coding":82.91499999999999,"Data Analysis":78.03333333333333,"IF":60.120999999999995,"Language":72.567,"Mathematics":87.20075,"Reasoning":85.64425}},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.55938095238095,"score_display":"73.56","source_stated_rank":45},"run_fingerprint":"29587f2cc5c720f70628452499241912fa62eff9d071ce6352d75349ded3f2f9","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"46ae306f5500a5ed0fbf09403b474f4997f972debae48bc9c7bf66e6cb53d456","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (max) · hle"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna","upstream_model_label":"GPT-5.6 Luna (max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2f6f8d9ce3b94eeeaea3d2b381f04047a9cd2860aa84db51286b1dcd0834e383","result":{"confidence_high":41.560332,"confidence_low":37.439054,"sample_count":2158,"score":39.481001,"score_display":"39.5","source_stated_rank":null},"run_fingerprint":"ad93cc86a26aee9067176fb1148207bff02db6b4d13617d096c7527bbd8b2a90","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a06bbcf69a9f88012549b82b3e8d5e99e89f06f641de213703dfc62b1dedbea5","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · hle"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-xhigh","upstream_model_label":"GPT-5.6 Luna (xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"63866e2e6122bc5bd637c502a103f2c54add59e5ca11a1314c3b4b6e7bc68373","result":{"confidence_high":39.036951,"confidence_low":34.966695,"sample_count":2158,"score":36.978684,"score_display":"37.0","source_stated_rank":null},"run_fingerprint":"0ddf8426e945589dec9280ba6b18a6068c6a3e44a8d782761e103fe9c8221334","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e57527d5ff03cb982977678154aae690ce1116129a9222e261bdf0953470ac3d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (high) · hle"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-high","upstream_model_label":"GPT-5.6 Luna (high)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8b27167844aef03b070169adfae4425353789f6ffecc3e3b1ff3a5f7838fc355","result":{"confidence_high":35.428594,"confidence_low":31.451495,"sample_count":2158,"score":33.410565,"score_display":"33.4","source_stated_rank":null},"run_fingerprint":"3b378b7fc3cfaf9087012a2814934c72e2d379155f887f647ddc9fc8d24662da","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (high) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1b05adb1361afad0b1f9e8620dc23129806fed91d0f832f8db40c53749d24d7c","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (medium) · hle"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-medium","upstream_model_label":"GPT-5.6 Luna (medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ddda09c5f4b3a057f4343234354194f874f2f2244e5950056bbbad89b4c43341","result":{"confidence_high":27.651744,"confidence_low":23.963583,"sample_count":2158,"score":25.764597,"score_display":"25.8","source_stated_rank":null},"run_fingerprint":"11067d57e5e2f591e6e5a9477dc870fc7da4849c0096035a3d1bdd5b6f4ede71","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"63812c1f8ccb739c604fcaec23198241726bd363d4bc5bbc0685a321e1229544","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (low) · hle"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-low","upstream_model_label":"GPT-5.6 Luna (low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"62dc1b37347f7ebbeb29f9ee9f460064932cecbe59e66bd2a648dfad6cf23f18","result":{"confidence_high":21.568524,"confidence_low":18.205047,"sample_count":2158,"score":19.833179,"score_display":"19.8","source_stated_rank":null},"run_fingerprint":"d7ce0793ecfbbf619542b7404397a72c05b9c4841d8d77e373e20f6d663c703c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9796bd8a89aeda75fb7fc70cc676f70a4f44a700675abe5e37117acbb3f84608","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · hle"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-non-reasoning","upstream_model_label":"GPT-5.6 Luna (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2bd2108a5935c0c05e7e73d3b25ea306706992f7e6e73a5d8192612bfd80068b","result":{"confidence_high":8.399222,"confidence_low":6.210619,"sample_count":2158,"score":7.228916,"score_display":"7.2","source_stated_rank":null},"run_fingerprint":"b48640a049a46a6fb913e09446575678332553097dd638e095daf7f1870bc43c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T23:44:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T23:44:20.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ccab513849b013a6c2ad017ab8baf20ca77b321b12015b019ec4d36b63925ced","metric_details":{"best_score_across_scorers":"0.41","model_release_date":"2026-07-09","stderr":1.5560917136921657},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CeDghdBk8CqtiFSbUzxNH5","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d410be56e6859f4e7081a0cc8ca85150541da72b651a27b3d98ed6bd89c2c16a","result":{"confidence_high":44.04993975883664,"confidence_low":37.95006024116336,"sample_count":1000,"score":41.0,"score_display":"41.0","source_stated_rank":44},"run_fingerprint":"77a20655aea156b9c0be971cf94967d6be44040472abf4261d102fe4a49e94d6","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-23T03:39:14.683442Z","basis":"evaluation_at","evaluated_at":"2026-07-23T03:39:14.683442Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d2682571b87318ba8ca6d69d9c976f5d3d04a58b3337777451a24e1d08ab2989","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-5.6-luna","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"84c79144605b5e27776cdf2a1fb96f9a0877c672b19821a0779b87b5753692a7","result":{"confidence_high":42.163772964055305,"confidence_low":35.9925396611952,"sample_count":null,"score":39.078156312625254,"score_display":"39.1","source_stated_rank":18},"run_fingerprint":"4f47ff07429b0d2a5e87349393986b1cc0349e2f99561bf8aec3e5b699c77cfc","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b46a11ac44d08172dead16ce","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7f378b481f3e2a88258212dca79bfcd7e7875cce24e2e727e1ec48953522d4db","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.010616,"date_is_proxy":true,"latency_seconds":53.521,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.736},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":92.424,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"max","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-luna","zero_shot_accuracy":90.909},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"e4d2635141aabcbfd1df9c6bc5e8cf793e330c8ce734c828665cbcd5ccef51da","result":{"confidence_high":95.06855999999999,"confidence_low":88.26344,"sample_count":396,"score":91.666,"score_display":"91.67","source_stated_rank":18},"run_fingerprint":"62b1c9156f49e947d4a1b4ebfd1815ef5838f3f463b29d9a0d12fe60c9442f2b","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2e8a7231cbcf763837cd370e5e7ae25a0f8b8e69a98dc44b7972603f1e45704d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (max) · gpqa"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna","upstream_model_label":"GPT-5.6 Luna (max)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"e7998a6261738f59e4685d80d293545f32a3c8b5a4a6ba07206225575248d126","result":{"confidence_high":94.331954,"confidence_low":86.325353,"sample_count":198,"score":91.111111,"score_display":"91.1","source_stated_rank":null},"run_fingerprint":"8770c6a3835b3930f3a085503e8813478ddd185b2b13fa515c11cc5863984c18","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (max) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"acae3213c2b4ef87de6f6bb19ecc943a5962d61dd7bcd60f6e47f9cd22e77a6c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · gpqa"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-xhigh","upstream_model_label":"GPT-5.6 Luna (xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ecd6d59e917acae16305589779095dbe7d7e76628a6d681f255bd6b0e1f61af0","result":{"confidence_high":93.039607,"confidence_low":84.446897,"sample_count":198,"score":89.494949,"score_display":"89.5","source_stated_rank":null},"run_fingerprint":"04af3688df6eba142a2883d5ed8ae672beece03bb1376ad6bc859398d2865787","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f358f29b688de88da59ccec77228b502f5287c4e95f977984b03c5cfb91a6e0d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (high) · gpqa"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-high","upstream_model_label":"GPT-5.6 Luna (high)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b34004c13c9e9b57aefb270b393371869e19c984ec74ecd21672d7847370f31b","result":{"confidence_high":92.793846,"confidence_low":84.098133,"sample_count":198,"score":89.191919,"score_display":"89.2","source_stated_rank":null},"run_fingerprint":"ebf978d111a6dfdceaaccf0ed2c1c1ada7367093e87a89859dc5046a78eb6649","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (high) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0c16f5dcda7f4bba662b5c494b3209ce93384b21dc7065f1610b0495253242bf","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (medium) · gpqa"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-medium","upstream_model_label":"GPT-5.6 Luna (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"32b3f6a9367ab438a41e90185361029553eb0209ffa229df50be1787bc699e4d","result":{"confidence_high":90.031469,"confidence_low":80.320728,"sample_count":198,"score":85.858586,"score_display":"85.9","source_stated_rank":null},"run_fingerprint":"78befee4bc9d1676c4022c0f201ed18700b852297adfdebb101c734d95fbc39e","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bbfabd513797705c6504221d24066924078c8524fe3dad56326f8a7013804d57","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (low) · gpqa"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-low","upstream_model_label":"GPT-5.6 Luna (low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ab821fd7a8963fc6d7d9b6f943fbcdecb6638f3c7185fc51da5c1c2c893b184e","result":{"confidence_high":88.053114,"confidence_low":77.741054,"sample_count":198,"score":83.535354,"score_display":"83.5","source_stated_rank":null},"run_fingerprint":"3f52b8188c9d1b91dea76c34dbc4e2f3fc76ed4794ef9fdca7c09a56af087e8a","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"87b8e98d67ba04f072b4c7679f4dbe6c6d67c997b528cf91ce3ae662ae3706c4","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · gpqa"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-non-reasoning","upstream_model_label":"GPT-5.6 Luna (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5342cadffc8a786a9978d3fc85824a6b9e737ff410b31b21421aeb34e6eb1204","result":{"confidence_high":70.874048,"confidence_low":57.663182,"sample_count":198,"score":64.545455,"score_display":"64.5","source_stated_rank":null},"run_fingerprint":"5235fc63006aff4393c8bda5cffd3d891dc4a8cee3d61427e1102b2d17265cab","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09T13:36:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T13:36:49.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c59e8ac1d473d42099a0d3fefe6273a1debe3468c3ea5952930746dc586cf64d","metric_details":{"best_score_across_scorers":"0.9160353535353535","model_release_date":"2026-07-09","stderr":1.728284355494035},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LWkDtkmd3TNmLzMhfZD5HW","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"0bd83982828cc0b1f2b5212843ad5c24bfcd4a1e84ba2f2ec0a2ee21f67f10f4","result":{"confidence_high":94.99097269030366,"confidence_low":88.21609801676703,"sample_count":null,"score":91.60353535353535,"score_display":"91.6","source_stated_rank":30},"run_fingerprint":"09c1387d1de134100267afb3d564cf98a05508d897ea557afe014cbe159fc81c","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:08:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:08:31.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4a8b069c1634eccb1a4341f952a4a596f90da01b7db1818ff33b6bc265858e8b","metric_details":{"best_score_across_scorers":"0.8232323232323232","model_release_date":"2026-07-09","stderr":2.717875263904491},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"nW39ZzuDQ3xTAGVoEfFG95","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnW39ZzuDQ3xTAGVoEfFG95.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/nW39ZzuDQ3xTAGVoEfFG95.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"5c5cb667daaab310f1cc0d5822739b6b5ec43596253a87576b11807d05c67931","result":{"confidence_high":87.65026784048511,"confidence_low":76.99619680597952,"sample_count":null,"score":82.32323232323232,"score_display":"82.3","source_stated_rank":114},"run_fingerprint":"b3d0be427ba276d5d1ea6a8da13fb242169d6f2bff061dde050dc70df68ebf89","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:35:08.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:35:08.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0659b50f764bd43953a93a1429bcd42185e2a158673489cb03d295df9af04fb8","metric_details":{"best_score_across_scorers":"0.6363636363636364","model_release_date":"2026-07-09","stderr":3.4273086529999337},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bDKKx7zTdW5WwMCfwqsLi4","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbDKKx7zTdW5WwMCfwqsLi4.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bDKKx7zTdW5WwMCfwqsLi4.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"84b3b8974d188445f2ca33e1cebb585e225e75f08e786e08d757e18a02ea99b8","result":{"confidence_high":70.35388859624351,"confidence_low":56.918838676483766,"sample_count":null,"score":63.63636363636363,"score_display":"63.6","source_stated_rank":205},"run_fingerprint":"68f9c75216f9b64f04bd7f240ec240867fdadaed2a3092ec1cc393222a36ddd7","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":81,"at":"2026-07-15T20:02:46.26298Z","basis":"evaluation_at","evaluated_at":"2026-07-15T20:02:46.26298Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fbb7091e7d1af985708a5ea5e16034edd37c25702a55d2c949778aa398d3e8f1","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-5.6-luna","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9819a714bdf0b96fdb894e3c4c2b63eec78f59faa3a2078687f18b9d0bcd93b5","result":{"confidence_high":87.19046705729694,"confidence_low":76.44589657906671,"sample_count":null,"score":81.81818181818183,"score_display":"81.8","source_stated_rank":17},"run_fingerprint":"9d60153ed18af4431adec8a8354f15a32e7bdca744b86716b89f5fb54db00db3","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_587496103eba42c84259406f","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f2f8eb675d5a4d14d203e84f","split_or_window":"provider-reported-protocol","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"06402a2375f4a77bf50b51c8f9f17c8069dbae1b9be589ef3eb06e044675583c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Academic · GPQA Diamond"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort and tools not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"b72ce4576adc1201eec7ec882a588d449d9c2f5cda949167eb39a584f0bce769","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.3,"score_display":"92.3","source_stated_rank":null},"run_fingerprint":"f5fbded0edca04b766ea7bc7890ec4c1058b65482a17ce1bde7c228472d8789a","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Academic · GPQA Diamond","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f89621a2390b533c10022bf","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"49c467e5deb322813087e1f920770314ac7388d1e63e0d7680aab5bc7cc35708","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.002842,"date_is_proxy":true,"latency_seconds":16.948,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.35},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-luna"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3bbe490f97552c9af77a72b06e989f3d33647223ab29de0a1a1fd606b2cdf480","result":{"confidence_high":86.72200000000001,"confidence_low":85.35,"sample_count":null,"score":86.036,"score_display":"86.0","source_stated_rank":50},"run_fingerprint":"7520166a1499737df38641036e51cd7ee412a83790681a56b8265a656e846e25","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T13:36:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T13:36:49.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9ea3cfbf4a6f942a9210dfebb864fd6fec00049aee3756813934f9296368b6fa","metric_details":{"best_score_across_scorers":"0.9833333333333333","model_release_date":"2026-07-09","stderr":1.1650421590325524},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"AhtQ5ggs4pTfLDqHFjueGy","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0308d85f90369d2eee9ad2f5f23e877c1b2ad27b956eee9188603f10586fd690","result":{"confidence_high":100.0,"confidence_low":96.04985070162952,"sample_count":45,"score":98.33333333333333,"score_display":"98.3","source_stated_rank":21},"run_fingerprint":"aa87e7605a746b56539d339c62d348cae9b62c5477cb16d836b10348589a5cd3","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:02:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:02:15.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"85d09e600f41d3fa4c3a5b67f6894bbcd551b293763c8dd38baf8e0a48d42bb8","metric_details":{"best_score_across_scorers":"0.6666666666666666","model_release_date":"2026-07-09","stderr":7.106690545187014},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ZgU8BagFmaj7BE94y3JDZA","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZgU8BagFmaj7BE94y3JDZA.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZgU8BagFmaj7BE94y3JDZA.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6e94ff28d4d28c54db2f895165b3b2081e04f004e67b94f4ccc27358e3aa5ad7","result":{"confidence_high":80.59578013523321,"confidence_low":52.73755319810011,"sample_count":45,"score":66.66666666666666,"score_display":"66.7","source_stated_rank":149},"run_fingerprint":"84301624dc023f87abdccab68fa8eb2570428f9d7ce53e601c14eb2c8a6123b2","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:20:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:20:15.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c23d9ef0209650539ad9226ac7f32c83d926997b883409e4607e03896b2382f0","metric_details":{"best_score_across_scorers":"0.4","model_release_date":"2026-07-09","stderr":7.385489458759964},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"QYZ6wRuT7WMkaYPKc9dRPy","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQYZ6wRuT7WMkaYPKc9dRPy.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/QYZ6wRuT7WMkaYPKc9dRPy.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"788cf320b8b4dab33ffafa6e4e179907254abcf2facfde5a4bbe82a75b41e1ef","result":{"confidence_high":54.47555933916953,"confidence_low":25.52444066083047,"sample_count":45,"score":40.0,"score_display":"40.0","source_stated_rank":206},"run_fingerprint":"f00f53637a5fbf16233fa26864c6adaabb756bc0ef42231a733c03990b8060df","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T14:18:35.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T14:18:35.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c44eda32aed6674907f99b0b2d2dcb79138a6b5d188eaec6d7927929c5fbfd5b","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.8210526315789474","leaderboard_private_problem_count":285,"model_release_date":"2026-07-09","public_example_count":10,"stderr":2.274515950332741},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"YLAk5J3Wd8EruVXee8bAQb","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYLAk5J3Wd8EruVXee8bAQb.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YLAk5J3Wd8EruVXee8bAQb.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.6 Luna (max)","thinking":null,"upstream_model_id":"gpt-5.6-luna_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"4b8b8ad290dc029c8d200cb56c5ff20f2f2dad01315a27e7ec1afd440109a271","result":{"confidence_high":86.56331442054692,"confidence_low":77.64721189524256,"sample_count":285,"score":82.10526315789474,"score_display":"82.1","source_stated_rank":14},"run_fingerprint":"6ec9853b1c7605c6909e11f522b17fdabfe769cfbe8dcf0572736d03ae346242","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-29T11:07:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-29T11:07:20.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"48e9763a48a4553fe08c170f34aeba011c6c52a52674988cd410a44328a3e499","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.41403508771929826","leaderboard_private_problem_count":285,"model_release_date":"2026-07-09","public_example_count":10,"stderr":2.9227738239445764},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["low"],"epoch_id":"ZryXhNUpPedPvGhbqdNBEw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZryXhNUpPedPvGhbqdNBEw.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZryXhNUpPedPvGhbqdNBEw.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.6 Luna (low)","thinking":null,"upstream_model_id":"gpt-5.6-luna_low","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"8529a72919ed4e10df2a23bee13a8a4b6dad794b9557e294720bbb011e813381","result":{"confidence_high":47.132145466861196,"confidence_low":35.67487207699846,"sample_count":285,"score":41.40350877192983,"score_display":"41.4","source_stated_rank":56},"run_fingerprint":"41199014bc81f3bfcb2a0fb30d3ed367af20d755bb8656b16ea5e4ff605a3f2a","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-29T11:07:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-29T11:07:39.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"069d0ee846b259b43e1b84d53550124eb4b3a2f63a91feade87a6b7cda8b0474","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.39649122807017545","leaderboard_private_problem_count":285,"model_release_date":"2026-07-09","public_example_count":10,"stderr":2.902681701641007},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["none"],"epoch_id":"YwHKNFQjbat97wVLqachmi","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYwHKNFQjbat97wVLqachmi.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YwHKNFQjbat97wVLqachmi.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.6 Luna (none)","thinking":null,"upstream_model_id":"gpt-5.6-luna_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"54f954333b94fd85ed6d1255a1f660d7b16d116de79037cac56a98ad7f18ddc1","result":{"confidence_high":45.33837894223392,"confidence_low":33.95986667180117,"sample_count":285,"score":39.64912280701755,"score_display":"39.6","source_stated_rank":57},"run_fingerprint":"1441abd5e5bc3423f1478a537cc8a9a4d0b800fb90a877621bb7092a326e711d","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7a48421a85a2e73a76d6dee4d272452ea25544dc959dff8b46c8a4063eee6ee1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.17671712499999995,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna-2026-07-30"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna-2026-07-30","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-max-2026-07-30"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f1bd525dceb8715c61fcc62b3b1067943be8d659f1b593e5d8d640e1fb897110","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.583333333333336,"score_display":"60","source_stated_rank":79},"run_fingerprint":"e79be6b80bdeaa9a1476fdd4a167ce31e2852494524a8dea5a60a6e975e91bb3","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"174bf57ea1ab014652fc22ce87d68185a918ebbe7a56a982fff7ebe00d12cfc4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.67,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a25f52167f074f4c6fc0b8292822583a158ac8829723c615e11795a7c03a5114","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.540000000000006,"score_display":"60","source_stated_rank":81},"run_fingerprint":"a2135032af8b6f74b89ec80970375efbc7fe40f607fa828237232f0b19408875","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"366f5ab62bee434d99a40efc673b1dc083ffccff521616678e1d753a82345b68","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.54,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b873ab74a6952a7311265def04cc5640c12ec82a9dd6ac72bf3f63e7aba1f0d3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.64,"score_display":"48","source_stated_rank":97},"run_fingerprint":"d06ebef5ed9ad07bd17d60333926179374462842115764838fc349ac16472b7a","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0822d8c1834dfc631d3ed1a0b87175e29c757f242777d5eb37ef998664e0c170","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.13048743000000002,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna-2026-07-30"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna-2026-07-30","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-xhigh-2026-07-30"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c1a8c23fd9fd4abd79b6adfe92c323cf48ab730f849640586feef3f8c4e1cb00","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.472222222222214,"score_display":"43","source_stated_rank":100},"run_fingerprint":"4076711a1067da841410d2fe849602bfb6537b5ad969131f94c405780dcf71f0","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ea891847bf663a7a298f661b9ad609502782032cf54e4e52fb71f9487a10b884","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.06253926000000003,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna-2026-07-30"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna-2026-07-30","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-high-2026-07-30"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9c75496257acd1fbd8a9f4417428457a769420660a4d4332fa80374cddcf3b57","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.33333333333333,"score_display":"33","source_stated_rank":111},"run_fingerprint":"fe51eeb232abed5688272412015e7792d2ff94ce573ce79e8cabf43c3fd0a9ac","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"13e3a7d96be0a0a4294e76d50c8634477143cefff3cadcaf1af941d46b7af484","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.29,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"48b73da3e3bc39313e0a200c14e954ebfc9ab3f13f9e94502817d9a06a49d5bb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.310000000000002,"score_display":"29","source_stated_rank":120},"run_fingerprint":"d1a91c3cad0d50f9a1cd45f62d134e80e63d35f36048d1481d2953d4e732c055","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"caf7d763962ab942bf7f486055af6711ec7a006195215071be303e9bba517711","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.02209704,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna-2026-07-30"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna-2026-07-30","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-medium-2026-07-30"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7a364a1d54b6e89deb0836a3061a26de249a5bb03cacc5cdd1111821bef0fae8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":7.36111111111111,"score_display":"7","source_stated_rank":150},"run_fingerprint":"ab1711397504ac63c3c831dd2abd55273e8c92961282452a2cea9decaacddc8a","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b93a27400bb9412465bf76e4462da3d924f84a0dd6ce151d26134895fed09c51","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.11,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7c34c3a29e763030249df77b8a0b3e60cfc861ce88e88b0c52da7da0e66319c3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":7.359999999999999,"score_display":"7","source_stated_rank":151},"run_fingerprint":"3098ecd8fe2ab336ef2d163b9071e45577c5b2aa4c34a8948a26f7bea84e00f3","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7469823b6eef90012158fd1862344b0cd84b9c992146d24d560c5c89c992cf0a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.07,"model_release_date":"2026-07-09T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6d153108e8e37983d7bc5ca0d86131839d2b02c860ffa03453be3baa89822388","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5.140000000000001,"score_display":"5","source_stated_rank":163},"run_fingerprint":"55532fd9db3058830b241950be28f007064b47b55744c8546e66204f443c69e6","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6ade4dfbc8ccb62698f746b6d7b8b1f574d39daa9c71aa9c76e3318c54a08d84","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.013629859999999995,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna-2026-07-30"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna-2026-07-30","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-low-2026-07-30"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"53c84c9bf60a7f5bc38f21bf51e9100bb0164eda8caffc1ed799d52155d32b4a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3.6111111111111107,"score_display":"4","source_stated_rank":179},"run_fingerprint":"aeed7858df96be67608b8546b72f7776b58416be0abc2933a20a5472a669351c","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b8627828244895c846cc8500646d520f2f8c3d78793e5d3544848951fb1f35a0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.008186350000000002,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-5-6-luna-2026-07-30"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-5-6-luna-2026-07-30","model_type":"CoT","upstream_model_id":"openai-gpt-5-6-luna-none-2026-07-30"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"13266d04e93c59bd686a19451482486754cf9cf3cb18fab80d29bd56b90d3b81","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.8333333333333334,"score_display":"1","source_stated_rank":226},"run_fingerprint":"77c9c22f62a4e0323b978858415388680f5137b0d8bef6ff0395bd5893726a3e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c75b22034b928af884dec0093b8b85f87261ba6c77db018f8eec948e8521fd5d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.67","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (Max)","source_effort":null,"source_model_version":"gpt-5.6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"873319a42d5eeb9de110535891773dbf67c9023ac36717153142b491a3062c36","result":{"confidence_high":64.483607,"confidence_low":54.394938,"sample_count":360,"score":59.540000000000006,"score_display":"59.5","source_stated_rank":83},"run_fingerprint":"038a2c8b9be35fe2f124ccb02994e59627c4cff6a757f35d53ea0bddb7980fae","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e81b85ce6676ab681305c3ee6d252c14e6a1922cf1065563bcd22d32aceb5176","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.54","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":13,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (xHigh)","source_effort":null,"source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"70b88c759f903f87bf3b7b4c377c0cdfebcb7a352a4bec2a9401401a4ce74262","result":{"confidence_high":52.796966,"confidence_low":42.53287,"sample_count":360,"score":47.64,"score_display":"47.6","source_stated_rank":99},"run_fingerprint":"c8182ad0c8359b020d618069ec8603a56595474fdac5e4bc6c5de7e779a20cfa","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7df779c43e7c4ba82b9a7f94ab2052d7e4c30457f2c263ca9fdd434910627367","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":14,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (XHigh)","source_effort":null,"source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"649cf450d872920936f053aa684ad8e9a3fb0a7b71c14cddcd3bb4112c26aad8","result":{"confidence_high":52.796966,"confidence_low":42.53287,"sample_count":360,"score":47.64,"score_display":"47.6","source_stated_rank":99},"run_fingerprint":"face05525f128a2a0364ab0e56a99342bf908939447a10ebbcf93dd462280c04","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c1ca59ce78234cc0698fee618874a7538b8d4b1067a9faeb05c66c43ee28de70","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.29","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":15,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (High)","source_effort":null,"source_model_version":"gpt-5.6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"cc5cc021a1b387b89c34fa594efbd08470844eb6d79563bf27e99af1ea7c0427","result":{"confidence_high":34.210762,"confidence_low":24.846146,"sample_count":360,"score":29.310000000000002,"score_display":"29.3","source_stated_rank":119},"run_fingerprint":"51d4320b66d92bbc2c621165152a98142129c9b8fd988542072db4d00397410d","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4b60915a272f6cfb2828cb0dd1a212d517114d7555ae78c12bea6a3dbee9b0e9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.11","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":16,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (Medium)","source_effort":null,"source_model_version":"gpt-5.6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"94d8df56d26be34222e0f0f1d3c65dbc9d8ea98aeedae3d4b2df340d09168752","result":{"confidence_high":10.530829,"confidence_low":5.089595,"sample_count":360,"score":7.359999999999999,"score_display":"7.4","source_stated_rank":146},"run_fingerprint":"0146b705828addd614a0acf53bf0ee7fcbedb6da94b1c52f596faf0f0d36aa84","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"324ae22f7ba30e3190eb16acfbffb0042fd3c4e2f4978bfa5dc1890438edef4f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.07","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":17,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (Low)","source_effort":null,"source_model_version":"gpt-5.6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ac57b5ba70a75f36d20c8bbf597243d0c0fcba14966488808e72f173238519c5","result":{"confidence_high":7.931503,"confidence_low":3.2958,"sample_count":360,"score":5.140000000000001,"score_display":"5.1","source_stated_rank":159},"run_fingerprint":"4806bd71474e5c28591484d6352602d2607f61b42a9688f950e0a899a6a6d739","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T14:18:35.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T14:18:35.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2d5acb28fa6ace6f26c9a78d0963311ddf5c0a7ea13de592edd973ba432229ce","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.6097560975609756","leaderboard_private_problem_count":41,"model_release_date":"2026-07-09","public_example_count":2,"stderr":7.712872341874097},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"NLiiJJSyA9wmg2FHENFpAq","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNLiiJJSyA9wmg2FHENFpAq.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/NLiiJJSyA9wmg2FHENFpAq.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.6 Luna (max)","thinking":null,"upstream_model_id":"gpt-5.6-luna_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"940a93eb7d331a9efe6e8ebe8fca4d3617f421d867cc2d2dab2b94a82cf24e2f","result":{"confidence_high":76.0928395461708,"confidence_low":45.85837996602433,"sample_count":41,"score":60.97560975609756,"score_display":"61.0","source_stated_rank":18},"run_fingerprint":"1a3c2af92c8c018814aa19821b6e76bbf3452a50d255ae7e45353cfe6424e83f","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1e1af07429dfcc7d72f9569533813e98bb4cbae4a629843a1191b5084747ae7c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.006419,"date_is_proxy":true,"latency_seconds":35.102,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.858},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-luna"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"278d2c6f2f215583eecb8d489f57eed89228b7526846da147efa24b0eff37b41","result":{"confidence_high":86.71068,"confidence_low":83.34732,"sample_count":null,"score":85.029,"score_display":"85.0","source_stated_rank":25},"run_fingerprint":"45ef2f46276df1a6b253ade305bbe22f900a0e65186333603045c1632cb0b8e8","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cf735371c3c5bc54e9c177c77be093a2e7aec4c2632686d2dc08303870a09956","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (max) · mmmuPro"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna","upstream_model_label":"GPT-5.6 Luna (max)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4bba8b7d5f4b770ade9767cff232718622e440913e3bf13cae7fd108a8e72319","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.554913,"score_display":"78.6","source_stated_rank":null},"run_fingerprint":"0e3a7d1eea8a9f3fef070735da4d26ba6fe86a837584c38087aca6707de5f5af","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (max) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"160fd0230d3977f28a69c4bfd0209df632f5aa8ca0e273c9bf44b418c147e193","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · mmmuPro"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-xhigh","upstream_model_label":"GPT-5.6 Luna (xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a269117080cf4423ad53022e836ed968bff03b8b1164fb5aaf2876e0de933f26","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.554913,"score_display":"78.6","source_stated_rank":null},"run_fingerprint":"b260a418cf7920400db1166b312026dd1af0707e5bfffd96e66cb039920607d8","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3f9e515c5722cfb4277af843a2ade5f408be20e672591e56f05760752304e5fd","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (high) · mmmuPro"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-high","upstream_model_label":"GPT-5.6 Luna (high)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"3e438674c171affd1ba166578f1b01853d9d2b66e51eaa5b4423c5169af4ee15","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.572254,"score_display":"77.6","source_stated_rank":null},"run_fingerprint":"25a7172fa74a7dd4a20942b9cf009cf742327680401ad6a805b77abb23bd1868","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (high) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a600aa9a8c2479774202b367a6c5a883bad6407ced29eb9796e76a0bb356541d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (medium) · mmmuPro"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-medium","upstream_model_label":"GPT-5.6 Luna (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"72156433e29842538a673b0051ef507713f93dfc6d5487e35f179b2b380b814f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.83815,"score_display":"75.8","source_stated_rank":null},"run_fingerprint":"a468cd8e40d17923f9f547be7c0ce5e8d7f1913e302f2ea0a6488a275aa2ce16","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5e20f7bb38c804f61a7e4aedd1075afaf4b90b2fd0059a6ae4c8983d7fb8c059","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (low) · mmmuPro"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-low","upstream_model_label":"GPT-5.6 Luna (low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"55e0f0a7eeee7cb8727dc8db9270de1fe3fe5fe653a11f15ad07ed1109b33de3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.988439,"score_display":"74.0","source_stated_rank":null},"run_fingerprint":"900e0da9e4e8840d55bd9316cfa18fedfd08265e8cb9a41de3cb2b3c435a1bf3","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0c5fe561daf00d8a531e35f8fe117e3cfbdbd9f1df0a193c96849450f6344954","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · mmmuPro"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-non-reasoning","upstream_model_label":"GPT-5.6 Luna (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"65f7d876d984eda666b0f9a4893d44a8f50510b79c1503aecc890aa164345a51","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.346821,"score_display":"60.3","source_stated_rank":null},"run_fingerprint":"bc68268bc056810d91d8e478371269c2416eb688654d0d205644ab1f85456217","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_39a5bff9a48a29bea5ce760d","split_or_window":"standard-no-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"587ace91886e374abb5a5d1bb358938aaaec870a3f84d098be5c77669460555a","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Multimodal · MMMU Pro"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7668a8bdde63b0a1b008fd1140ee89a7608afe0b024f589d244f127d83c35ea3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.4,"score_display":"78.4","source_stated_rank":null},"run_fingerprint":"299c3b3e0294ec1b625ebfd431b490711d0456fccbb548ea125a96a7b1a4f80c","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Multimodal · MMMU Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f89621a2390b533c10022bf","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_aa0096763ba73cd85e46674d","split_or_window":"standard-with-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"56cb61a20fbfb9139b2d2666e7d04755507e58aefeb91275ce2358bd7fd95772","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Multimodal · MMMU Pro"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI provider evaluation; effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"tools enabled"},"run_count":null,"scaffold":null,"tools_allowed":"tools enabled"},"protocol_fingerprint":"c9869f62f1ab1e482460e86f9c0ac73683c41febdcda4cd23712ea1098430062","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.5,"score_display":"79.5","source_stated_rank":null},"run_fingerprint":"68ee796baa8a96caa02981ba475c9ed2ca9582491ef3b7af9e9d7af46ee32313","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Multimodal · MMMU Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f89621a2390b533c10022bf","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f3e5a54c0e4c7faa61960ef37beac4cf8cea4450a3f4c87d0f38c64e4aa70312","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (max)","source_effort":null,"source_model_version":"gpt-5.6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b167fa5fff6078b29a21202603c406fa525b8f2b62cf3e1f5d32f4813ed2e92d","result":{"confidence_high":31.363827,"confidence_low":12.800153,"sample_count":71,"score":20.571428571428598,"score_display":"20.6","source_stated_rank":51},"run_fingerprint":"f2af66f812156fbb9a314f00b152ac45a62cf1e17f5df39809d47c2e017c262b","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f8c82a6486e6acab72b7fde2cbb486355f8311fc8bd235990b5ae61d759951f7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (xhigh)","source_effort":null,"source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"21f7adc95b9519ab1c59e0c3445b51066d8d0963d5083b6499526ad53f3cbcdd","result":{"confidence_high":31.363827,"confidence_low":12.800153,"sample_count":71,"score":20.571428571428598,"score_display":"20.6","source_stated_rank":51},"run_fingerprint":"d48c0411d0a14c6f604fa4b92e8863544c897a07a10ab3a6469d97e764279136","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4d601574c3caa495666bb96d7ad11b1ec144e529f5cca8520db78f8115c8607a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (high)","source_effort":null,"source_model_version":"gpt-5.6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"fc738e47c880d92ab1a5c3e682a22657da32bbf60a164d10a147a542a837797c","result":{"confidence_high":26.884354,"confidence_low":9.690263,"sample_count":71,"score":16.5714285714286,"score_display":"16.6","source_stated_rank":72},"run_fingerprint":"37eeee20f0d3105fe012f9769fdec22eb5a1ecac327aa4e45497675d23e71ab5","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"512cb74d27741b4a9d25b2dcec285711f9bb4b769480bf61666121731de42a4c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (medium)","source_effort":null,"source_model_version":"gpt-5.6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"9b46c884b13e46ac5eb7aedb24688b830eab2c15c69a6e6d764e88b2ee4aa299","result":{"confidence_high":12.56782,"confidence_low":1.780808,"sample_count":71,"score":4.857142857142859,"score_display":"4.9","source_stated_rank":111},"run_fingerprint":"2529d8d43510e7c635995731f6705bbf86fa2be8f2981b626555e3ab6abcf422","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"de3e4005120d73599519c1b537269a0caef3d29db18fb4c670ef06a88dc81395","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (low)","source_effort":null,"source_model_version":"gpt-5.6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"75b1d10c7decf388ceebbe3e97fb88946f1a82c3ba7128416d12d482b55e1ec7","result":{"confidence_high":9.340258,"confidence_low":0.671592,"sample_count":71,"score":2.57142857142857,"score_display":"2.6","source_stated_rank":129},"run_fingerprint":"18d3b0806c7a9ffcadbae649a0beff55587897312a7fecf53447e3ba246eee89","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"beaa2452a5f300e10c380ae0246927f8c8431069d8ace071d837a33cc7d39ef6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (Non-reasoning)","source_effort":null,"source_model_version":"gpt-5.6-luna_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"307114605cef494e2bc3407e498b0ef213c2bfe7c142538f7b43263ec16caa77","result":{"confidence_high":5.661392,"confidence_low":0.013679,"sample_count":71,"score":0.28571428571428603,"score_display":"0.3","source_stated_rank":156},"run_fingerprint":"9f80f3067cdd314e22cb28411ca783f54d01b0d838f8a9ba9a7723481d6c8ccc","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dbe3175aaa9dfc55535cbfc2778ee77d0136dd774af162b18e84c43c79913dfc","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (max) · critpt"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna","upstream_model_label":"GPT-5.6 Luna (max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"bc9491b05f93a870678accfb8323e22b073ac66644b468e2ad857206018de36e","result":{"confidence_high":31.448598,"confidence_low":12.756296,"sample_count":70,"score":20.571429,"score_display":"20.6","source_stated_rank":null},"run_fingerprint":"bdf6a1b9b5594640fe7de2cb01530f0b8399082606c361c7e584db59789d4af5","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0f4cea367bc70e040ee0acc093ef20d83001970ac48e2eb2382fe42536783e55","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · critpt"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-xhigh","upstream_model_label":"GPT-5.6 Luna (xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"44d8168ef43c8f544576635cdfda90883d5c7256523b4a96854070233cf5d03a","result":{"confidence_high":31.448598,"confidence_low":12.756296,"sample_count":70,"score":20.571429,"score_display":"20.6","source_stated_rank":null},"run_fingerprint":"121a76b5b68166942ed53244c9ed168251c3916fc807d4ff21022adb7e89e77c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"abdc46e70fac67d2c18a27f4bfae08e67cd06d9cedb1f29e998014ff675edb03","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (high) · critpt"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-high","upstream_model_label":"GPT-5.6 Luna (high)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"717baa3ce6c1049bdfbfffb74f4440fd34faeab7ab47f381d60b1a1ee24e2bf0","result":{"confidence_high":26.967943,"confidence_low":9.65315,"sample_count":70,"score":16.571429,"score_display":"16.6","source_stated_rank":null},"run_fingerprint":"6b4662a10164c620f95a6afebaa979c4fbd69bf0496b20d9b36f083c73d6a311","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (high) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a5c17bd6f30efcecac73cd81fa9694938e29827c93ac2ba404a792ad15435f02","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (medium) · critpt"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-medium","upstream_model_label":"GPT-5.6 Luna (medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"615bcca6a398c8953502c3ad62137da7e3434c5c5e9f48886d4b143e2af87175","result":{"confidence_high":12.642381,"confidence_low":1.769008,"sample_count":70,"score":4.857143,"score_display":"4.9","source_stated_rank":null},"run_fingerprint":"f8d103bb8d6bebca31f706e21c776558da1823d91a93de8c1c2a85b7a47d49ef","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"223639888553eb5bf5df5a5fd34018c54446d8c7f8f7a1985227acc8317c7c4f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (low) · critpt"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-low","upstream_model_label":"GPT-5.6 Luna (low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"025ff918c6c4f3433e5c6a8a9adf152541d45052da208b01a48eeea3d4cf061e","result":{"confidence_high":9.41179,"confidence_low":0.665999,"sample_count":70,"score":2.571429,"score_display":"2.6","source_stated_rank":null},"run_fingerprint":"03a7e4b50e5ee5c7bf43f48775efe682f6909d258195dca4499b3f405cbf54be","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"46c792e1369eaf0bf98101882cdb9892eaf3c01533a76c94ef184cd197a3c9b4","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · critpt"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-non-reasoning","upstream_model_label":"GPT-5.6 Luna (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b44a40160d1e2cf9db103bbc99bcb445f56b80653caaf9638c9771d25f23a130","result":{"confidence_high":5.730683,"confidence_low":0.013504,"sample_count":70,"score":0.285714,"score_display":"0.3","source_stated_rank":null},"run_fingerprint":"01c65d51c452f8f12f92a43b7d9a172d1226e1ebe35628c5656b627c7bb2b912","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f9f8ae895abd79c2299d4f5b6df8550d439809489ecac15d5d0221814d230008","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"1648b670dd9e109a8ca13ae1a8d575f070c45134efc8688a0b7f8d6e92abd9cf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.8,"score_display":"46.8","source_stated_rank":57},"run_fingerprint":"0c08a5196a3350f18a0eb69ac4721d17a5cf2a8ef9871535312b9f21e9af40a1","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8af6ca7f9746d0e75f3207cd8478dd44be21c4d7f43f841dfd53902639178b00","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-luna_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"7361fa7804b70cf192451d3191f324482051d9c62c1bdf33a5cb6a4c8875ac5a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.800000000000004,"score_display":"46.8","source_stated_rank":56},"run_fingerprint":"e63f4bc19c04d58157bb67787e23b4bf3364221c64ecf904c48b95594e11c819","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acc1809f725b1fa2144abbdc","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T13:36:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T13:36:49.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"817d3217af415e7e3e1664eb9c38ba6f06bcdb5eaa6ff3b123d5469d522c3de8","metric_details":{"best_score_across_scorers":"0.4","model_release_date":"2026-07-09","stderr":4.92365963917331},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BJiQ8aRjMyCGPTBGYfBhyj","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c2a69be84ed5ff0d0aedbf84c51e799dcc720d40cc9d9be22b6169874239f968","result":{"confidence_high":49.650372892779686,"confidence_low":30.349627107220314,"sample_count":100,"score":40.0,"score_display":"40.0","source_stated_rank":24},"run_fingerprint":"dd7b8bfc5ec4cf2bb973a53f45804aaf6b76639277cf5a5804fa13fb58ac374b","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:16:10.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:16:10.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bc98913ffe46d6e1939b0245ee419de7c591e9cede926096662235167641a96a","metric_details":{"best_score_across_scorers":"0.21","model_release_date":"2026-07-09","stderr":4.093601807403323},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"UhmACN5spAPyAtWWBf4MAP","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUhmACN5spAPyAtWWBf4MAP.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/UhmACN5spAPyAtWWBf4MAP.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"73a3d7d91a4b0724388db6ea9ee505c9ff438852aae6fd53befe8bfd6f1d7286","result":{"confidence_high":29.023459542510516,"confidence_low":12.976540457489486,"sample_count":100,"score":21.0,"score_display":"21.0","source_stated_rank":80},"run_fingerprint":"98c690a0f72979dc32af398cf805992ed3b632309e7bdb55945b7c6ed77587e7","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:35:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:35:15.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"90b1e77a5413232efb7b3a4ed037f0eb7b5ae79fcbca1cb6a03e398d49315d94","metric_details":{"best_score_across_scorers":"0.02","model_release_date":"2026-07-09","stderr":1.4070529413628967},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"d9jMGFp8BwebW7sufdxbnC","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fd9jMGFp8BwebW7sufdxbnC.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/d9jMGFp8BwebW7sufdxbnC.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"fbacb753a4d6be4cfe36d72363827dfc816f006a6c730e205b0f125bb556b960","result":{"confidence_high":4.757823765071278,"confidence_low":0.0,"sample_count":100,"score":2.0,"score_display":"2.0","source_stated_rank":185},"run_fingerprint":"264186ade39ce6ab237792e47c403286917c9ae23f6ac3928ee45058bb2798de","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2c53df1587a119db1dcc1cfe6ce5699b606824d05ed2340e3baf60b5b92dd6b1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.32","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (Max)","source_effort":null,"source_model_version":"gpt-5.6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"833b5eb9853713ace12830a8bb02a8f907235c2ed7576989e479d4f57ffbb594","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.0,"score_display":"88.0","source_stated_rank":80},"run_fingerprint":"b14737d5355f273a9f1033d4dafa99fa0d3e9f7076f8e61622f1202ddbf3d38c","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"74edc36a95bc29ebc7203d7674de779cef5bd012f125f3c66f780c2eba110560","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.21","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (xHigh)","source_effort":null,"source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ad077ae5f00a47a89323549299c4c59c1e97b9c0e4273b6804c38590c2e872bb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.67,"score_display":"87.7","source_stated_rank":82},"run_fingerprint":"e13a28334088305db6b796bc037dd21a3477bf23d279bc4e87b87d1ef1e0ea68","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1ab9800c8adec5f49a7193a08a1c7f7b5a05f80dcd041f6216aa5e65d9cbd691","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (XHigh)","source_effort":null,"source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"2a3745ac207727ed95d855860a5a290f2a9b77519f6aa278f0a619ba10135cb2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.67,"score_display":"87.7","source_stated_rank":82},"run_fingerprint":"6fb6c7d1004ecd8a0620d67f22ed866509c57a6c5b084538b4501ba387c06151","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"174539855585801b304ce4c3ec9894737ee0ed108123580f6e6c36e9669a10b7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.13","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (High)","source_effort":null,"source_model_version":"gpt-5.6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b6a058cea692d38676209d4af247e16483c12b5c8c508ec8793c8a42c31bcfc3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.5,"score_display":"76.5","source_stated_rank":119},"run_fingerprint":"af70a01a65fe8c823cc3428efa5b0c5eed357ff0d7ca9fb7fa6cbf35b6cef419","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f93d5d1a7983d74b2a2017e011982cd79c9958822170e56c65ef8fafb0da56d0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.06","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (Medium)","source_effort":null,"source_model_version":"gpt-5.6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0e95738b875e0df8e23e93dfa98f68aeb4bf4630fa4d84bb2a4999f15961c296","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.49999999999999,"score_display":"56.5","source_stated_rank":159},"run_fingerprint":"8bd8495784eaac7e959108fa5c1a1e8be1525fd3bb86a5c59827855c9af6ab76","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4ee5b0c7cbc53ee88f4b23d91ec4716b93cfabd5d7c33facbe2fa3777fa6f06b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.04","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (Low)","source_effort":null,"source_model_version":"gpt-5.6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"9c8072ede66b3735f36c94a6d12c3b05905bf33db3e464d458ebe8f07c19e600","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.17,"score_display":"34.2","source_stated_rank":192},"run_fingerprint":"9c3204e43069733af83ee6b005c249052b23315f283c780ee5d29c1552f759fe","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"168910b98c87b2d4f329c0f67ff85337c6b1cfeeb5803eed1d2412c845e7b7ee","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.042712,"date_is_proxy":true,"latency_seconds":201.075,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.142},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-luna"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"20496d624d7d324c8ef09f05cc494dc0bc3c962587e693faf78cf7d10819a7fa","result":{"confidence_high":95.23832,"confidence_low":90.76168,"sample_count":500,"score":93.0,"score_display":"93.0","source_stated_rank":9},"run_fingerprint":"f60f8ef81ba02c33abd185c547b0c918cdfe30475923966f630467700978a034","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5f3ea02a4a93099996523bccdf15507986235643d6f8618a5cf22be9d7d9e07e","metric_details":{"model_release_date":"2026-06-26","pass_at_5":59.45945945945946,"potential_contamination":true,"sem":1.4693249036306386},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"contamination-risk","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"GPT-5.6 Luna [medium]__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"4bbed7a27d62a359466c297ea94c4763d7e66439d93473a4ab91adb70a3a02d3","result":{"confidence_high":46.48348041471965,"confidence_low":40.723726792487554,"sample_count":null,"score":43.6036036036036,"score_display":"43.60","source_stated_rank":9},"run_fingerprint":"d6102f84c0897c12091cbef601a473d78c6f0d573d85b911dce232aea03f70d6","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"contamination-risk","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"c2c4746f44933525a55f0f6be757ec250707ca5f933a1407f614ae238ed6ff99","metric_details":{"ci_attempted":448,"ci_half_points":3.992001235671055,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":301,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":101.68080357142857,"mean_cache_tokens":14666893.714285715,"mean_cost_usd":3.028116810267857,"mean_duration_seconds":1122.9470243995536,"mean_input_tokens":15443718.330357144,"mean_output_tokens":73399.70758928571,"median_agent_steps":92.5,"median_cost_usd":2.2920917999999997,"median_duration_seconds":982.9724034999999,"median_input_tokens":12278493.0,"median_output_tokens":70253.0,"median_output_tokens_to_pass":68234.0,"median_peak_context_tokens":201647.0,"n_attempted":448,"n_passed":301,"n_tasks_attempted":113,"n_tasks_passed_any":102,"pass_at_4_points":90.2654867256637,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_luna_max","source_model_version":"gpt-5-6-luna","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"b4b94369640b3c39e71eabb0fdb8f097be0b63c1b2cab8ccc91fc7e330628e9d","result":{"confidence_high":71.17950123567105,"confidence_low":63.19549876432895,"sample_count":448,"score":67.1875,"score_display":"67.2","source_stated_rank":21},"run_fingerprint":"b05c3659f52b5854b8ae228d95f895551d8cb65c05c044570ea2f16fd45302fc","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"f75cf9efbaaade244ad49fb0681666af642cd7819261dd96fd20a7d56ba5f88c","metric_details":{"ci_attempted":452,"ci_half_points":2.1681017528097977,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":257,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":71.09955752212389,"mean_cache_tokens":7096152.3539823005,"mean_cost_usd":1.5356192685840708,"mean_duration_seconds":731.6846703871681,"mean_input_tokens":7610577.53539823,"mean_output_tokens":44677.900442477876,"median_agent_steps":63.0,"median_cost_usd":1.2449332,"median_duration_seconds":608.986619,"median_input_tokens":5550213.0,"median_output_tokens":42374.5,"median_output_tokens_to_pass":41546.0,"median_peak_context_tokens":138126.5,"n_attempted":452,"n_passed":257,"n_tasks_attempted":113,"n_tasks_passed_any":89,"pass_at_4_points":78.76106194690266,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_luna_xhigh","source_model_version":"gpt-5-6-luna","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"902612b84ffc73f9e3a7ffe41b32682ca2d68d109a80288ee9d9e0876a46e9fa","result":{"confidence_high":59.02650883245582,"confidence_low":54.69030532683623,"sample_count":452,"score":56.85840707964602,"score_display":"56.9","source_stated_rank":38},"run_fingerprint":"baa4ef52f69dac26b0af13ce5a59ec0d4fbf20cbaaec680ed986dc5c294dd393","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"f9401bbca60c7f7678ed072e34234e1d87b10bfcf72034c36ee2c9f067da8b0a","metric_details":{"ci_attempted":452,"ci_half_points":2.9195672479165315,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":200,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":49.02212389380531,"mean_cache_tokens":3055611.469026549,"mean_cost_usd":0.7779007499999999,"mean_duration_seconds":473.83880740929203,"mean_input_tokens":3372477.7876106193,"mean_output_tokens":25778.274336283186,"median_agent_steps":44.0,"median_cost_usd":0.6635542999999999,"median_duration_seconds":394.07379549999996,"median_input_tokens":2538825.5,"median_output_tokens":24817.5,"median_output_tokens_to_pass":24022.0,"median_peak_context_tokens":90166.5,"n_attempted":452,"n_passed":200,"n_tasks_attempted":113,"n_tasks_passed_any":85,"pass_at_4_points":75.22123893805309,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_luna_high","source_model_version":"gpt-5-6-luna","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"f5d2758e1b4e78f6e6db101abd9d10654b85e41ce83cf7387636b3e9552784dd","result":{"confidence_high":47.167354858536,"confidence_low":41.328220362702936,"sample_count":452,"score":44.24778761061947,"score_display":"44.2","source_stated_rank":55},"run_fingerprint":"8c0770758fcfc89224069006801f59d2956edddea1c1c1f16a2f3eec916a8cf3","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"676a6dc735bca60dfc18cd9121915e023ef73c4604820642852e32f1a2570810","metric_details":{"ci_attempted":452,"ci_half_points":0.8303197562056515,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":51,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":23.67699115044248,"mean_cache_tokens":500661.2389380531,"mean_cost_usd":0.21630978982300886,"mean_duration_seconds":181.5252541438053,"mean_input_tokens":617827.4800884955,"mean_output_tokens":8179.570796460177,"median_agent_steps":22.0,"median_cost_usd":0.1994702,"median_duration_seconds":151.8156975,"median_input_tokens":517201.0,"median_output_tokens":7918.5,"median_output_tokens_to_pass":7414.0,"median_peak_context_tokens":35812.0,"n_attempted":452,"n_passed":51,"n_tasks_attempted":113,"n_tasks_passed_any":31,"pass_at_4_points":27.43362831858407,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_luna_medium","source_model_version":"gpt-5-6-luna","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"11e913e1a6aadedc26c858015cdce3122c59e25943457af35699ad9a524b7e15","result":{"confidence_high":12.113505596913615,"confidence_low":10.452866084502313,"sample_count":452,"score":11.283185840707963,"score_display":"11.3","source_stated_rank":69},"run_fingerprint":"54f324c86a68ade354b2ff2bf4183027fa26e3bf47a54b2e6eec6e1f7553a46a","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"dcd51b0a20e6b209b4ba3d2c7914643ecdf1cd039d00327277874f35e835aab3","metric_details":{"ci_attempted":452,"ci_half_points":0.8303197562056515,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":7,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":12.464601769911505,"mean_cache_tokens":107102.01769911505,"mean_cost_usd":0.07240623938053097,"mean_duration_seconds":77.14656007522123,"mean_input_tokens":150031.52876106196,"mean_output_tokens":3127.754424778761,"median_agent_steps":12.0,"median_cost_usd":0.0697851,"median_duration_seconds":69.9232685,"median_input_tokens":143435.0,"median_output_tokens":3059.0,"median_output_tokens_to_pass":3036.0,"median_peak_context_tokens":17490.5,"n_attempted":452,"n_passed":7,"n_tasks_attempted":113,"n_tasks_passed_any":5,"pass_at_4_points":4.424778761061947,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gpt_5_6_luna_low","source_model_version":"gpt-5-6-luna","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"bb3681ea6bb4394317456d941bd363758524dbdade35109186822addcdd1cfb4","result":{"confidence_high":2.3789923225773326,"confidence_low":0.7183528101660299,"sample_count":452,"score":1.5486725663716814,"score_display":"1.5","source_stated_rank":70},"run_fingerprint":"eaa2c3a423ba70fb3e40931d513b7950073951204f3a40d1746bc26530bffd78","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0a8ac3db80a4468dbf7a441361f331c48729686174b70b5a13d677e94e8f55e6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"101.68080357142857","mean_cost_usd":"3.028116810267857","mean_output_tokens":"73399.70758928571","model_release_date":"2026-07-09","notes":null,"pass_4":"0.9026548672566371","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-luna (max)","source_effort":"max","source_model_version":"gpt-5.6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"1f09c874eb23359f948d0d50235a72ea26afe03383ae1338051bcb46bb1567c0","result":{"confidence_high":71.17950123567105,"confidence_low":63.195498764328946,"sample_count":113,"score":67.1875,"score_display":"67.2","source_stated_rank":21},"run_fingerprint":"be48ddd154cb92fd21799e05fd3465d646bc2cd9e438fc84bb88bede897732c7","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f7e5627a33b136ab9feff976e528f635c7122c96173d5666a7fc0fda655e025d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"71.09955752212389","mean_cost_usd":"1.5356192685840708","mean_output_tokens":"44677.900442477876","model_release_date":"2026-07-09","notes":null,"pass_4":"0.7876106194690266","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-luna (xhigh)","source_effort":"xhigh","source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"269cb78e0e8e58dd50d18a3a53f962a4739e2cab5730988af2fb88053818973d","result":{"confidence_high":59.02650883245582,"confidence_low":54.690305326836224,"sample_count":113,"score":56.85840707964602,"score_display":"56.9","source_stated_rank":37},"run_fingerprint":"ff829efff24bb20b0b6175d9fc61da88403048a3f0793a6d1e09dc93f73baa53","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4fb9ee12dac7efe742c199a99edf612fb974dff5c2e15063c7d7b376ec080cde","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"49.02212389380531","mean_cost_usd":"0.7779007499999999","mean_output_tokens":"25778.274336283186","model_release_date":"2026-07-09","notes":null,"pass_4":"0.7522123893805309","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-luna (high)","source_effort":"high","source_model_version":"gpt-5.6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"1e71b3e4eaa4fe4784ba8b03867a4d315749a2797ce14b0ef0a0dd5c0d9d455b","result":{"confidence_high":47.167354858536,"confidence_low":41.328220362702936,"sample_count":113,"score":44.24778761061947,"score_display":"44.2","source_stated_rank":53},"run_fingerprint":"849b02b88088d16675aad12329fb7b214bc2c3ca1c50f11d8524b66138c66226","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b9482444e4d1f2d6cf83b2d2177df1e7ed0ee5cccff965e816de295fa38714a4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"23.67699115044248","mean_cost_usd":"0.21630978982300886","mean_output_tokens":"8179.570796460177","model_release_date":"2026-07-09","notes":null,"pass_4":"0.2743362831858407","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-luna (medium)","source_effort":"medium","source_model_version":"gpt-5.6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"9cf4d56b5597b79a9fc9824e993c54481873096c6811382c1f9494a0401356e5","result":{"confidence_high":12.113505596913615,"confidence_low":10.452866084502311,"sample_count":113,"score":11.283185840707963,"score_display":"11.3","source_stated_rank":68},"run_fingerprint":"bf68abd128de00777820c59ca5c7b12b2a08a8c764f9529926b39542bc6a52c4","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9f6fc2ac22ed388169a66b24cef2e6c9ac214db9f5226f62f8e18713081e79cb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"12.464601769911505","mean_cost_usd":"0.07240623938053097","mean_output_tokens":"3127.754424778761","model_release_date":"2026-07-09","notes":null,"pass_4":"0.04424778761061947","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-6-luna (low)","source_effort":"low","source_model_version":"gpt-5.6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"c41158c50ba4d1d25986f5646c9b4b7031ea6505e02aaf9b6ddc5899168018e5","result":{"confidence_high":2.378992322577333,"confidence_low":0.7183528101660299,"sample_count":113,"score":1.5486725663716814,"score_display":"1.5","source_stated_rank":69},"run_fingerprint":"28ea24c249773918214ba58e290cff132855947d2463ca06e5bfb7b3d3c2f368","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_e361abe5266a3ddfdcef017b","split_or_window":"openai-provider-comparison","unit":"percent","version":"DeepSWE v1.1 · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"eab709db7489b9aff14115d336ad3b7ca82b2bc9cee997de00e5dff86a579816","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · DeepSWE v1.1"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value for DeepSWE v1.1; competitor rows are secondary re-reports; exact scaffold, task coverage and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"coding-agent tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"28bd89a57c6784aa44dda4f48bb1ba1e048300dc4313759e24d750d50b4e533a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.2,"score_display":"67.2","source_stated_rank":null},"run_fingerprint":"748f926b86315196a53e02eb9f2aa91ea34141e5f049a05e22a3b9a0074813ac","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · DeepSWE v1.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f89621a2390b533c10022bf","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_7664765571d731e0f6ff7adb","split_or_window":"openai-provider-comparison","unit":"percent","version":"SWE-bench Pro · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"46ca57698b406ec228041c008f1c2944a064309096b231121c2e48822eb0cf80","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · SWE-Bench Pro"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; OpenAI rows are provider results, while competitor rows are explicitly retained as secondary re-reports; exact scaffold, task revision and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"bb784ac6361cbbe179556c67a1258b8773e5fdda82afa9a06e9f85b5d8df29c2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.7,"score_display":"62.7","source_stated_rank":null},"run_fingerprint":"77771af20b5b3da8f68091d5a4cfc96ad8f718dd8ab0f83193e165f14133d11c","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · SWE-Bench Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f89621a2390b533c10022bf","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6c2e458419745597fc0d4094775fd8a5b64d50db9774c33c8a72d0878f3537a7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.03","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"208","tokens_per_task":"87284","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna Max","source_effort":"Max","source_model_version":"gpt-5.6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"b406a1aab9f551b0fafde6de8553408d66a80c1444d2f57932a589a8d60ba3d8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.9,"score_display":"35.9","source_stated_rank":33},"run_fingerprint":"81cda4a4e956a3a4efe1f5c8441552fb766590c7fb75e79080928de317122e33","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b8ea484a12e8fa146fb355cb8c3abb60c250d666b97a473c1a223ad64037a434","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.44","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"98","tokens_per_task":"40598","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna Extra High","source_effort":"Extra High","source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"bb1cab7ea990b38e1f06e69deda0856623313d87b4dbf9b7a558e86c83edf7e9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.0,"score_display":"33.0","source_stated_rank":41},"run_fingerprint":"49a7849b8a1dca06b2a0b9fea2acec5c218b2893643246095be817674cc63c49","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e57ae0eb08739254a7d234eacf19fe91194cdbc0e54dbcf4425cd21f9d3b77cc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.25","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"64","tokens_per_task":"23368","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna High","source_effort":"High","source_model_version":"gpt-5.6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"160525a7224b0cdf8ace5b01538e038f1ef1e3c618220d0ed39781527160b6d4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.4,"score_display":"29.4","source_stated_rank":47},"run_fingerprint":"2ee8ba23cb4a357679983394b21d885a8850dffd71620ecbfdb46c322540af11","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"99ec0e26c236a91c98e34dd00014abe97319b1cf2ab81a846c4bb231ca2e0e30","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.08","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"32","tokens_per_task":"7642","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna Medium","source_effort":"Medium","source_model_version":"gpt-5.6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"5058e75417b665b4cd42086949a664717c6ffa6484e2e08fb2460b02c753da39","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":22.2,"score_display":"22.2","source_stated_rank":56},"run_fingerprint":"ce1b44903648e0659dff529304328a4f0b99935c578db995162426e876728a3d","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f4bb4be94d7ba2d469ee50a9cd337e6a712d6d7b403e71287a1bf22fddacc2c5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.03","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"steps_per_task":"18","tokens_per_task":"3288","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna Low","source_effort":"Low","source_model_version":"gpt-5.6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"7dd79042288b874c3cd14ee2eaf2dba13bdc277f53550cc20a1cf5dbb3931028","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":16.0,"score_display":"16.0","source_stated_rank":57},"run_fingerprint":"c9e24206ddca89262c800ba35f9531639629d0f2c9d43c5b0628db78104263d3","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e1d8c08138007b1f04cf6558749f34d029abc09b18e1830afaacddd2555531e2","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna","source_effort":"max","source_model_version":"gpt-5.6-luna_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"codex","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"97f8a238f9c83cde2459efea5678c908db5f00a8835b0048d51e297bd9df0c56","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":39.81,"score_display":"39.8","source_stated_rank":23},"run_fingerprint":"0a0459249532b0f961583af32298e359e1b0ed4da17dfd9cb777d4d3b8d4bbca","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"77aea82b75968223e8715925bab949c98a7055d0d0bc1e528d0ad832a5f0d75e","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.57961,"date_is_proxy":true,"latency_seconds":4175.412,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":11.613},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-luna"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"2ee7fe3cf47843a491bfc8dafbc215e827c40771140c768526157f3b93399881","result":{"confidence_high":84.53948,"confidence_low":39.01652,"sample_count":null,"score":61.778,"score_display":"61.8","source_stated_rank":14},"run_fingerprint":"39553872e43c18ab44013e62fa199b4144b00b3f62acfb690ae7e79567dec554","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fdf240e1cb4506d2cdc94c4d84d64c7ecb86f7bfcf453e9115ba5fa2a228c34a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (max)","source_effort":null,"source_model_version":"gpt-5.6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"7630d273ab869745648a519a0ca98ac5516baec25f5b9dd628b63297bf6e3572","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.587962962963,"score_display":"53.6","source_stated_rank":59},"run_fingerprint":"9d3d85ad6909e7c95f9da7dcafe6b3a9f33d2e9e36caf6567c7a72083d88b783","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b07eaa1608e70d7ea0a1f53d9e24a4e68520cb48083680cb3796993d7a43bfc2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (xhigh)","source_effort":null,"source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2ae20e8ac5a94b2c1693c651305bf2067d206eddebbe501a7b8b03b8b5becc97","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.0,"score_display":"50.0","source_stated_rank":91},"run_fingerprint":"5f6eab3e160ea2b75cdac4fde60ee303efa1be4176a836f70e600c46c7ca55c6","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cf18960363630dad7bd83868d43bc925036f42b494874f94194b5dd021aecde0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (high)","source_effort":null,"source_model_version":"gpt-5.6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"4948dfb48b45ce8cc51ad7b747539d00555d332da8d531519426be94a986642b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.6944444444444,"score_display":"50.7","source_stated_rank":83},"run_fingerprint":"52d0a4b6eaeec4bf5ee63d8daebd4ffd9971f15408c4c7bf3d9364fe826ccab9","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4cd255d19d9bdc8fbe0481ec1df909f29a359a3b915b07db5b878ae91309f5f4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (medium)","source_effort":null,"source_model_version":"gpt-5.6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"82ca1a5076c25e48b684b7c0b4fa1adb8cda99346d506999fa36c2294dc07bc0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.8333333333333,"score_display":"45.8","source_stated_rank":120},"run_fingerprint":"9b1e2045f878bd6aefeff2d565869ed5afd3b59e6c5f8e98a69ecc6fa58bccc1","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f337d0581f127e79f0c2e87fe604fb046063e6d54f9669de85562fe462fed50d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (low)","source_effort":null,"source_model_version":"gpt-5.6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"827fedf83db925a349da020dacaed3175e08f98884dec1b85943a1a16bef7a9e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.6018518518519,"score_display":"45.6","source_stated_rank":121},"run_fingerprint":"4bbb2093fd2438b9e3c95c2faa5f0c558d6b277b94997d88d38cc7aed51b4af8","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fe0a9d3217250ef691f70fcd77ff42c9cee1b0f2c21efbd3ab86f1836c10ac87","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna (Non-reasoning)","source_effort":null,"source_model_version":"gpt-5.6-luna_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0913ab093625e47d08d4e2390f7ab28299095137c0e8e1a7504c187c90cb17b2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.9305555555556,"score_display":"39.9","source_stated_rank":151},"run_fingerprint":"354b433c0db64ea4c4bd686bbec471b176aee4ab01bf47d1c681222c280bb67b","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"721154bd8882b0911e25d7f7e30386c6cccd087450ec5ecdd27c2ce0a1bfb8ae","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (max) · scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna","upstream_model_label":"GPT-5.6 Luna (max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0cf0e03f6252acc76a11c99418fb0c0fd66b05e9566598fb76e15bbe7a701be2","result":{"confidence_high":59.262712,"confidence_low":47.818755,"sample_count":288,"score":53.587963,"score_display":"53.6","source_stated_rank":null},"run_fingerprint":"ad6cc0fd7293eb29ea57957db2dfb394ca66bc046014b80b09f811f0f37b4306","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c35d1ea5557488ede9559dad6fd85bed4b220091bdd6ff4d20bf19258d80e3d3","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-xhigh","upstream_model_label":"GPT-5.6 Luna (xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"9d7665d887c79e983e5c4c70f3f7d7255634d6d18963f0cd964decf625fedd63","result":{"confidence_high":56.193199,"confidence_low":44.720539,"sample_count":288,"score":50.462963,"score_display":"50.5","source_stated_rank":null},"run_fingerprint":"7d6bd61175632241c9a28e024076c6b354ac38d2533c77366e9fa2c7563d203d","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f7e0ce9f53d6ca5f355756d96e0108813ce6c584da8ff9e94017559a0fd1ec18","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (high) · scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-high","upstream_model_label":"GPT-5.6 Luna (high)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b529c9334b1a7b146ef5572a36bbe459153a24fb488f27404ec41875caf06005","result":{"confidence_high":57.332639,"confidence_low":45.865442,"sample_count":288,"score":51.62037,"score_display":"51.6","source_stated_rank":null},"run_fingerprint":"69112c341535d29d4db65c171a0a51f75bb917e952cadbed3ba6b487295c7db3","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (high) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-07T14:33:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-07T14:33:47Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"90d7bc427755a6bf2a60df73458bbc9764ceca0d54fddb947f710558940bfa3b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (medium) · scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-medium","upstream_model_label":"GPT-5.6 Luna (medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"7a0be86d5f1cc4833ba5f4dd237a008516461e011e61d3b6dfc932cde5154703","result":{"confidence_high":52.526587,"confidence_low":41.077249,"sample_count":288,"score":46.759259,"score_display":"46.8","source_stated_rank":null},"run_fingerprint":"115f30b6db3ecdc2fd60c3a54f9ff9084a58b2b7efcc381c159c05784db22304","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bc732395573494d2bfd79ba6c06769d307189ede60bb3b93574f1e7c9a7534d9","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (low) · scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-low","upstream_model_label":"GPT-5.6 Luna (low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"90e61dee34fc1e173ace07fa081b6d204b02d118fbedaa60b9481203f586cedb","result":{"confidence_high":51.835627,"confidence_low":40.397603,"sample_count":288,"score":46.064815,"score_display":"46.1","source_stated_rank":null},"run_fingerprint":"47661b9dad0ea1d113dd8d423d0a6b981045be92f7895b691c17028372811cb5","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6976f2d1868583a241454679a3f91b0c03f4f0cb7a9c6473457763bd70459a2d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · scicode"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-non-reasoning","upstream_model_label":"GPT-5.6 Luna (Non-reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0a2a686775056feb81f86000431f4c8206ac4d06f9cd17cb33840d4221c70e5c","result":{"confidence_high":46.15109,"confidence_low":34.888855,"sample_count":288,"score":40.393519,"score_display":"40.4","source_stated_rank":null},"run_fingerprint":"4779987ba8c1d7f90e1a7c7b41afaf1f8e5380997360a736e753e1a651c8fee5","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a6af0e4938fcfa1bbd8a9b32bd7958267d44a967edc5b7b589b46921bfa1067d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.359171","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.6-luna-max","source_effort":null,"source_model_version":"gpt-5.6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"8970e59b1071083f0b89273e65b29adb3f889938ffb6a4156d776dbbfcfbbafd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1667.4,"score_display":"1667.40","source_stated_rank":11},"run_fingerprint":"c349c3ec03eee2644f01880f597e9e2e2efe1dcf58ff8c12c0adb19b02d32e6a","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"45e403dad94a9f5733975d5b414c45bb21e4971f5dd6e1db1e60af15f08d70bf","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.726426,"date_is_proxy":true,"latency_seconds":1551.831,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.069},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-luna"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"dd70dd00d3753779ce80cb1d81a51b7168ecdd4db3b79b9d658b43c73db42dc3","result":{"confidence_high":83.07024000000001,"confidence_low":71.03976,"sample_count":null,"score":77.055,"score_display":"77.1","source_stated_rank":28},"run_fingerprint":"56151422a54777db4490c0950e548e043977d6088e410e928944f1eee58d1597","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c7ac148be9f1bbb229fe3da90e96cbd723231e0fcd54508ae2f268c410005959","metric_details":{"license":"Proprietary","variance":10.698987063171021},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.6-luna-xhigh (codex-harness)","subset":"webdev"},"run_count":null,"scaffold":"Codex harness","tools_allowed":null},"protocol_fingerprint":"986dd9c9872d93f7a5d18acb135a2c4c5e560c14377aba050fb5e444c28ebc0a","result":{"confidence_high":1524.3865332744972,"confidence_low":1511.5647208868024,"sample_count":12776,"score":1517.97562708065,"score_display":"1518","source_stated_rank":47},"run_fingerprint":"e7014f8162087708fa56d3da52d604bf429e20b2c00beaca8a37b17255b596e7","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9f9c1b69351710d862f3fb79845a6e1f6af0b14dfc4cb85e2984f1775113a12","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"1e85cec03a3246c0aaa6770c34983a85610c18631767075cfd108bd1fc31ec83","result":{"confidence_high":1526.48,"confidence_low":1511.27,"sample_count":7828,"score":1518.87,"score_display":"1518.87","source_stated_rank":45},"run_fingerprint":"20c38d768e5e582ecc902f41177a3374b647fdd5f4c5d1a706d3100b6195ea2c","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4f21fa80f581221a32d68c0f3fad1967fac55828ea1e511fc36ff1b83eb7e5c2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"4c42f574a3937ca56a96171015b4b7db3075113a478e4f98ae2cbe31f2e26dbf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.86,"score_display":"60.9","source_stated_rank":47},"run_fingerprint":"f80c97718d29751a74df3c8ba8b2bc3abb3a3a44654a3c9a345401e1903a80bf","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f7ef9cb63a71726b3530338249187aa8dc45f946566373a3a17dcd148d1e3c5d","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.053767,"date_is_proxy":true,"latency_seconds":355.751,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.991},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.6-luna"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"242c3f4aa6e78a8049c35b4987ffa9f04d3e4642f71392cbdc5be64ba3890fe0","result":{"confidence_high":80.96835999999999,"confidence_low":77.08364,"sample_count":null,"score":79.026,"score_display":"79.0","source_stated_rank":11},"run_fingerprint":"433cee16268e3a3d4817bb41209554d1b2b41cf477b3c4d61ad413edc1856ab3","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cbf0334afa06f34bcf8ca7fe2ed1dcea20a61b643227521a46b3c6af1f6dc3bb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (max) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-luna","upstream_model_label":"GPT-5.6 Luna (max)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"46a543367488f07f749d13a9ed37969cffb343704d07dc2bafba29d1718f76f2","result":{"confidence_high":87.718144,"confidence_low":71.52254,"sample_count":89,"score":80.898876,"score_display":"80.9","source_stated_rank":null},"run_fingerprint":"c95ae78be6dfe5d92e6de310b0df662639acf82015d7c475a19b26f5418e5ea2","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (max) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c8ef579f9f2048f00c4c0d8eb2a48acce73f1096cb5a55f4edb96a72b49a44da","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-luna-xhigh","upstream_model_label":"GPT-5.6 Luna (xhigh)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"10ce7d1fd41b5bcbc1ebe8b678cb3e474eb35e78245f2465112923122a24d6b2","result":{"confidence_high":85.266453,"confidence_low":68.229681,"sample_count":89,"score":77.902622,"score_display":"77.9","source_stated_rank":null},"run_fingerprint":"516f388594db5a6b2f683a0f1578621032e08022b5fb388ab92d18955e6e0eac","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1c9cdcb50cccc5508b531c871e828c5e18da86266fab457cc924a0aa79649bb7","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (high) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-luna-high","upstream_model_label":"GPT-5.6 Luna (high)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"f18a02b6601557d00c956e14a9ba255b834ef617f6f3f36a967271507cb65ac4","result":{"confidence_high":78.235947,"confidence_low":59.46267,"sample_count":89,"score":69.662921,"score_display":"69.7","source_stated_rank":null},"run_fingerprint":"63729bcd080f4ac5bc9c29d56a4463c4818aba4ec3e6f9b3b7dab1b8496e4710","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (high) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9de5923a5ffecda0946e8807799510ce3733fd334cca2bd64edf2f03e37475fe","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (medium) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-luna-medium","upstream_model_label":"GPT-5.6 Luna (medium)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"eac29c2a341ac4a5799d5e08c7c674b2847e89cedf5cb5593c26cdb32bb9b2d4","result":{"confidence_high":63.202803,"confidence_low":42.900783,"sample_count":89,"score":53.183521,"score_display":"53.2","source_stated_rank":null},"run_fingerprint":"8a0dc9d75abd430529c14db5da057bb86619fa9ba0af89abe521c1dcc63fda5a","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (medium) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5d829d869d06a419c58b5c5238ae42a837ef024fb6ebefc3249e800fc763e398","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (low) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-luna-low","upstream_model_label":"GPT-5.6 Luna (low)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"eda096fdaf166d709988e7216f4d1366dceb41bf87412907a77ebd730ee15fd2","result":{"confidence_high":53.803572,"confidence_low":33.630223,"sample_count":89,"score":43.445693,"score_display":"43.4","source_stated_rank":null},"run_fingerprint":"151c7204f0aa69e5803b3357e0ff9aaec252c1360c3e28005816833798c1e55e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (low) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"33906de5c3704a55c46a154892644ad2f4c2bd334188403f36cd2d46a0da612a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · terminalbenchV21"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gpt-5-6-luna-non-reasoning","upstream_model_label":"GPT-5.6 Luna (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"7e1547895a7e02a2311e4201816cc01f70c778a72d932182d74d319bb778e2d0","result":{"confidence_high":49.338381,"confidence_low":29.478587,"sample_count":89,"score":38.951311,"score_display":"39.0","source_stated_rank":null},"run_fingerprint":"3fd8cfd6bda10b0af7d350846c5be554be53e92b9f0236b941e0abf1302de4ed","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_d522137a2b9b55b4156d183d","split_or_window":"single-agent-baseline","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"fresh"},"measurement_id":"dcb37286d8750b4c14d3eb8ae734daeec184b2bab14675bc99bb0ef7bbe194c4","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · Terminal-Bench 2.1"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"single-agent baseline; exact harness not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-09","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"terminal agent toolset"},"run_count":null,"scaffold":"single-agent harness not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"35ec4343c781d4675f2f64710c5000fe0977214013c6f6f70121275e1c4fe180","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.7,"score_display":"84.7","source_stated_rank":null},"run_fingerprint":"426557831e0d840dc27440c7f0bfd5162417c85f7d81e5b34a7d13992a39eb6b","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · Terminal-Bench 2.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f89621a2390b533c10022bf","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T21:39:34.661007+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T21:39:34.661007+00:00","status":"fresh"},"measurement_id":"81c952549a478f4ad4ffe2a7859898dac4e507f5f3357e10cce51cb3e3ed5ae2","metric_details":{"accuracy_stderr":1.25,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:07:07.601743+00:00","total_cost_usd":1597.5,"total_tokens":11872804231},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"OpenAI","url":"https://openai.com"},"agent_url":"https://openai.com/codex/","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"5b10f961-312a-48ae-83d9-d5ce4de60a0e","model_release_date":"2026-07-09","model_url":"https://developers.openai.com/api/docs/models/gpt-5.6-luna","n_trials":370,"source_row_date":"2026-07-12"},"run_count":5,"scaffold":"Codex","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"7de4f97a12e9399bd298fc0584c86bda697ffff8673185d4fc32c4ed36b239d2","result":{"confidence_high":24.059073,"confidence_low":8.102651,"sample_count":74,"score":14.32,"score_display":"14.32","source_stated_rank":11},"run_fingerprint":"dd0c64f78a79ef68fd9d3c1b6e28961c2be02ec9296d9f2456ed0633100e5b63","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"330b55172919ccb6ab02ce05cc18146bf28ce1be14e73052efff82b8f4e8c0c6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (max) · tauBanking"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna","upstream_model_label":"GPT-5.6 Luna (max)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"2f0500d763c3ca201f3996008fa7227c097c74f25144824b446e5861f5e4bba1","result":{"confidence_high":40.918924,"confidence_low":22.786532,"sample_count":97,"score":31.134021,"score_display":"31.1","source_stated_rank":null},"run_fingerprint":"c35590786701e97bb0997ee8421949356d8e9adf38e44d4f5d0d7e30f6758cad","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (max) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f16313f2e7a2ff69b97249366be320f7117878dfe9ff47db311a8085b9b5035c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · tauBanking"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-xhigh","upstream_model_label":"GPT-5.6 Luna (xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"659b98819cdb548d12ea82ea74092d441e03deb882477b65891d03e9a3a8a125","result":{"confidence_high":38.335634,"confidence_low":20.609881,"sample_count":97,"score":28.659794,"score_display":"28.7","source_stated_rank":null},"run_fingerprint":"6c1eb45ade0243a1037fd17d402e1485c4ab8f133d7c73d7cdd9dc9a021ed3fe","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (xhigh) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0a80f050d735e1bb57966d869e32b018ee42656adc92ee187e7180071450d2cc","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (high) · tauBanking"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-high","upstream_model_label":"GPT-5.6 Luna (high)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"b68fb7b8475b9d3715911b71df2cd28e5a5d4edf923eef73718d50ebb45cdbcf","result":{"confidence_high":34.622773,"confidence_low":17.579493,"sample_count":97,"score":25.154639,"score_display":"25.2","source_stated_rank":null},"run_fingerprint":"575bc78405f4843ed15e41f6c5dcec2d5014efec401de1a412c2814fc2cc5c64","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (high) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"146c0eaa2c6e8b3d9aa89fed9b7b45688bc450096010226fa53f94de42a08006","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (medium) · tauBanking"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-medium","upstream_model_label":"GPT-5.6 Luna (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"71b9ff54cfb0846cbc1e9488b01f16f5ecd09f190318d8be47391172d76f3a77","result":{"confidence_high":26.516594,"confidence_low":11.405851,"sample_count":97,"score":17.731959,"score_display":"17.7","source_stated_rank":null},"run_fingerprint":"12c10e546a5adcb7f6889a2aee6b500cd5cc0f63602cc2a34bea10e359466426","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (medium) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fb88cbbbd4d030f6e705bd1a6915e2232540d8c2f736b2c374688f2b2376ebda","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (low) · tauBanking"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-low","upstream_model_label":"GPT-5.6 Luna (low)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"cbd8e204259a50325d7b8a66f65b653d8dbaf9fe3104118b6c18220b8a8fca00","result":{"confidence_high":20.870905,"confidence_low":7.531658,"sample_count":97,"score":12.783505,"score_display":"12.8","source_stated_rank":null},"run_fingerprint":"2873de69227e228da6715116180bcd1c0a07e37ac9cf20bd5abc93016adb6ecc","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (low) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6d422fc55a2f9b50d87fa462bd2888472c4c71101652229cd4650256b464317a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · tauBanking"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"gpt-5-6-luna-non-reasoning","upstream_model_label":"GPT-5.6 Luna (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"511243e17e28a616835710b690c41aa6d35dd286bcc98a51b680701333092e7e","result":{"confidence_high":17.201075,"confidence_low":5.251564,"sample_count":97,"score":9.690722,"score_display":"9.7","source_stated_rank":null},"run_fingerprint":"edebf2e7f786c8719c7ae3ce88cf1e77a3a343f21665a8273c0599d5983b6142","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"727517de4e267b4e809cf6fb4958e1c33e0de1fc036a9f754e8e54fe49209030","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":42599},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 5.6 Luna (xHigh)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9ee2f3cd59b931730ec0d817da184c9a66da38b65075b52050c580ebff1f4639","result":{"confidence_high":-0.0049177854981589265,"confidence_low":-0.018391575882733734,"sample_count":4028029,"score":-0.01165468069044633,"score_display":"-0.0117","source_stated_rank":35},"run_fingerprint":"b301dbe2a649668bb344ceb1a10af0b18f2c0ea3999e6cb0ad0574046eb1119d","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"appworld","metric":"task_goal_completion","name":"AppWorld","release_id":"release_3fa145932216c0afcdd9dae6","split_or_window":"test-challenge-all","unit":"percent","version":"official-c1f56015cf7c"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":58,"at":"2026-08-08","basis":"evaluation_at","evaluated_at":"2026-08-08","first_observed_at":"2026-09-02T21:28:04Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cb4146b5c08b1dccee907b1c3991f2eaf2dd48142fc33471381bc2f0a19717a4","metric_details":{"interactions":11.17,"normal_task_goal_completion":85.1,"scenario_goal_completion":52.5},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"leaderboard_row_id":"1095b768-6513e0c6","method":{"name":"kecaipan capybara","tooltip":"Self-organizing context assets learned on train and frozen for closed-book evaluation"},"repository_revision":"c1f56015cf7c3441ff1933f5bfbec879798d7bbe","source_model_label":{"name":"gpt-5.6-luna","tooltip":"LongjiTech OpenAI-compatible Responses API; model ID gpt-5.6-luna"},"submission_url":"https://github.com/pkc000pkc/capybara"},"run_count":null,"scaffold":"kecaipan capybara","tools_allowed":"AppWorld APIs"},"protocol_fingerprint":"ef1dc06cb4f5d702032de3e7246e740e9fdbc27cb185f697fca3cb42f4d74a81","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.4,"score_display":"73.4","source_stated_rank":null},"run_fingerprint":"45c45e459ccf4e11c60e8a2f4bdae7f5c11801921418694f5cfa13b1f3d5a78a","source":{"content_hash":"d44080bb01254150fab30f845d22262a28ee7c90f7850b98ce0ab235ea043f0f","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-02T21:28:04Z","homepage_url":"https://appworld.dev/leaderboard","id":"appworld","last_fetched_at":"2026-10-05T12:33:29Z","license":"Aggregate server-evaluated scores redistributable with attribution (2026-09-02)","locator":null,"name":"AppWorld official leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/StonyBrookNLP/appworld-leaderboard/c1f56015cf7c3441ff1933f5bfbec879798d7bbe/experiments/outputs/_leaderboard.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ae9c2d1f81bc6fd257f16a98","provider_config":{"source_id":"appworld","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c15ca12d19d44670e72974eebfe9ca52b41250d20dda884fdb7369c8a9b74f0a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.6 Luna","source_effort":null,"source_model_version":"gpt-5.6-luna_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"93ba9825f0574e5d080f2afb2c579a469251701b6b0e8ec17f61ee00ebb8a257","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4094.7120000000004,"score_display":"4094.71","source_stated_rank":37},"run_fingerprint":"0850fd527c2cd5faa69fb0c228c6edfbf93fac08480e44ca6c3c14412b819c25","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e08847800071627e0e69db55","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"793bb7b504136ef541b626de990f2194419fadb7dd8fcc9eab865a31e5dfb74f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-18 / +18","source_locator":"Leaderboard models table · GPT-5.6 Luna (Max) · gdpval"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-luna","upstream_model_label":"GPT-5.6 Luna (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"00d4dcdb3715db3164d5734daf67ec713be5af37ebb06f2a47b72f192b6d3b48","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1463.23,"score_display":"1463","source_stated_rank":null},"run_fingerprint":"8f3f56f02509f4d0fc124703cc4d2b295603f0e902e328092416c04a85c261b1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_1b694cc32a697b0749e4e7f5","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7e44ffea88be3d1dfdc84f086677b8e15b2589e89c21915dfa20095c82a2982e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-5.6 Luna (Xhigh) · gdpval"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-luna-xhigh","upstream_model_label":"GPT-5.6 Luna (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"5410fd4c43eb32d519358e7f31f48e794cd733be0ca1d78dc6da199b25658fcc","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1383.71,"score_display":"1384","source_stated_rank":null},"run_fingerprint":"20c594030d6116060ccf5a36d31ee23aa346ba5ef2fce8456ec74d588394972f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_e57bf627bda67d6b38ba2302","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9dcb669c2e4cdbbc3cf946e1b746741e52b0eb815b8b20902526cf2b60b97f33","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-5.6 Luna (High) · gdpval"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-luna-high","upstream_model_label":"GPT-5.6 Luna (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"fb316eff28959a92d3588a424a7629feb3c00044e7118a58ca6cfab11035c1dd","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1318.76,"score_display":"1319","source_stated_rank":null},"run_fingerprint":"194a15c8e979903c6885591029d5a05846eb0b8ffb6d55f60fed5e64cfdc146a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0bc8d2a65f5e960dc81d9df3","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0b12d67ba5811c3c0dedd0fc152804a91b844dacb4d411d8af6c6083610a276c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-20 / +20","source_locator":"Leaderboard models table · GPT-5.6 Luna (Medium) · gdpval"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-luna-medium","upstream_model_label":"GPT-5.6 Luna (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"1fe893b12cc80ce41f4ae962f02096a1360e636278a609187c45857d50034c66","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1132.91,"score_display":"1133","source_stated_rank":null},"run_fingerprint":"274ff474b5c5594ea391c020599d9b9f8b5700d82dd55da1f2ee08777401759e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d00d7de512e3af291d4a5480","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"116fddce7c0fb0df5ba388ee62d3daec596dddef0746cd7ab109f238baf3adcf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-5.6 Luna (Low) · gdpval"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-luna-low","upstream_model_label":"GPT-5.6 Luna (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"d9dbbdb63982e7a731d7614c7d6db7a8d879612906e0b60a48385e94ef9dd88f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":980.97,"score_display":"981","source_stated_rank":null},"run_fingerprint":"eb867fffca1769cff17577cd19ed53b0a0ffb1137207effc99dd3b6e663dae2d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_51b7e18fac1fc5a685c88719","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e8a206574c6e1cb1bc467a093e402969e7c5203e5af50cbf136f6b347c2f77d8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-07-09","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · gdpval"},"model":{"id":"openai/gpt-5-6-luna","name":"GPT 5.6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-6-luna-non-reasoning","upstream_model_label":"GPT-5.6 Luna (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c546b7b67fe90181117b9c4ba3eaabb09d9eb8424b5e39eb4011077ce6adeade","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":906.18,"score_display":"906","source_stated_rank":null},"run_fingerprint":"ad23c80cfffa666a3f2114dd74b3d7037b604d658fdb7a24fc4b5ed115189257","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.6 Luna (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_30f8dc7382ba44e3acdcf843","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b1dedcf12a27a2062c04c21f0ecf873ab1f2dedc27f5d3012c5f37ea2ff74538","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-16"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Opus 4.7; model release: 2026-04-16","source_accessibility":"API access","source_model_name":"Claude Opus 4.7","source_model_release_date":"2026-04-16","source_model_versions":["claude-opus-4-7","claude-opus-4-7_high","claude-opus-4-7_max","claude-opus-4-7_unknown","claude-opus-4-7_xhigh"],"source_reasoning_efforts":["high","xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"14fe8dda082f1ad2da9384ea9bc8dbaebe129930d2038e14a99e91e608936a39","result":{"confidence_high":158.39,"confidence_low":154.37,"sample_count":null,"score":156.35,"score_display":"156.35","source_stated_rank":22},"run_fingerprint":"da2bc71f4b4587faaaedb78fad811050cfb5fa9fe45644cda2cad9d0ce1aac41","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bc89e06c89d745c77565b69e","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7dae1b320cc8eb6ae39aa7c0d6fdde8b515a6642757ae119b3f5f85bb61002b5","metric_details":{"license":"Proprietary","variance":3.698326141413234},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1505.0694003033816,"confidence_low":1497.5309740544262,"sample_count":64946,"score":1501.300187178904,"score_display":"1501","source_stated_rank":5},"run_fingerprint":"aaf388c193eb7db4b5c8dd23e1069b86ce0035291b0ab53ae587f84e54047d9f","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9a1e5cb7fef9371ee02f9d82c41b3b4014994721f6aa330b621a1ca4683e8f52","metric_details":{"license":"Proprietary","variance":3.6990862223031287},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1497.8684561327354,"confidence_low":1490.3292552739774,"sample_count":66058,"score":1494.0988557033563,"score_display":"1494","source_stated_rank":10},"run_fingerprint":"d0d8a67a2fb49689620aa7b3b49af895dd15a51d47d8fb1fd5b796dd46163b5c","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f0e4e36e0f79737649cef2f6","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"d7b740d1ddc4bb500f27b9034807234f9cd6f7b8b97f5a5b6a84ea851302229a","metric_details":{"category_scores":{"Agentic Coding":50.65633333333333,"Coding":82.088,"Data Analysis":78.26366666666667,"IF":66.7375,"Language":77.914,"Mathematics":92.85374999999999,"Reasoning":87.19225}},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.52935714285714,"score_display":"76.53","source_stated_rank":29},"run_fingerprint":"01f21f0b6dabee690a6ecfe4931a35be086e609781f7b94e50a03469ea2234d4","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_509db2bd77140fb589d8a000","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":165,"at":"2026-04-22T20:13:01Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-22T20:13:01Z","status":"stale"},"measurement_id":"6d66ea5f8256c1686d45d07c8657b7a6ef1d34e2527587667bc4d517aabc732a","metric_details":{"calibration_error":47.0,"confidence_interval_half_width":1.88,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=claude-opus-4-7; createdAt=2026-04-22T20:13:01Z","source_row_created_at":"2026-04-22T20:13:01Z"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"claude-opus-4-7","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d01400c08c62bdd550d08c7022773e5e4755a97a7cfa1d3e78bfc898488a7ed4","result":{"confidence_high":38.080000000000005,"confidence_low":34.32,"sample_count":2500,"score":36.2,"score_display":"36.20","source_stated_rank":6},"run_fingerprint":"ac4ea5bec2e704918121c3f91e259fd98555940b449d9686c1d73e96112f22c1","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=claude-opus-4-7; createdAt=2026-04-22T20:13:01Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_706026ac74b842322735028b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"36f2dbcadd8a3fc2a2e7f3041102655738643735e6cd4da8da9fbb28764ecc92","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.8.1, p.196"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; 1M total-token cap; no tools","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2","source_published_at":"2026-04-16","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d7d98f87fa41bd327e950f68c296f35ec13542f17afb43d0484123327ba02455","result":{"confidence_high":48.85949,"confidence_low":44.950023,"sample_count":2500,"score":46.9,"score_display":"46.9","source_stated_rank":null},"run_fingerprint":"7e86ae1fc2b6b2653736bed0452217d9f6f9d8c973a7775316339dc57bddd992","source":{"content_hash":"a3416021494c6882904f6a123b4a9d3761d9596b999d8356f9aaf644fcf7aa14","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf","id":"anthropic-opus-4-7-card","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.8.1, p.196","name":"Anthropic · Claude Opus 4.7 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c93fdc26da7ffe52c4cf6b50","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-28","status":"stale"},"measurement_id":"ff9d097e90b6abf6faf52c389034dbacc68f72b0d6efad733a83b2fec997e400","metric_details":{"comparison_warning":"Opus 4.7 value repeated in the Opus 4.8 report; freshness is based on the original publication of 16 April 2026.","comparison_warning_code":"rereport-original-publication-applies","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Table 8.1, p.194"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"rereport-original-publication-applies","judge":"vendor-reported HLE grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"HLE original full multimodal","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"0df5d061b6688c2d5839476620d66960011cb69043185dbed87ed1f68ab76d8f","source_published_at":"2026-05-28","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c9be347eb24d15402e3627321b88b6351f44c84c5dc5075dce14156ec34324b4","result":{"confidence_high":48.85949,"confidence_low":44.950023,"sample_count":2500,"score":46.9,"score_display":"46.9","source_stated_rank":null},"run_fingerprint":"a2447a4d6b92fe9c8e1e32edb03e987dc1ac36898054d6a087ccce9d2d09f33b","source":{"content_hash":"5bb1078972885f7ac1ba60fdc904e99944086bcb964e5c5d699e7eaa01abdbb8","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:30Z","homepage_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf","id":"anthropic-opus-4-8-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Table 8.1, p.194","name":"Anthropic · Claude Opus 4.8 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/0f0c97ad20d8005706296bd92aa1c27c6b2f4f61/Claude%20Opus%204.8%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":4,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"secondary-re-report","verified_status":"secondary-re-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"evaluation_at","evaluated_at":"2026-04-16","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"be5f777cff3b0b8b915da3fc2bf6f4b9d7366796390e2d99f1b4ae273c88dfda","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/anthropic-claude-opus-4-7","source_model":"Opus 4.7"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0dbbdead9c14ff96980b23bbbdba1d436b07a61f32d0ef18f6042c8f111b24f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.04,"score_display":"39.04","source_stated_rank":14},"run_fingerprint":"459c06c4ff4d69ffefd9349eba3a3de34bf35caad13c831a1ffb6a203b667b47","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0807140abc11307b3dfaf2f2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"a0df0efd617ed000e57f342c8421691c1b0ab46f2d21316da6c44776db22d81b","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.8.1, p.196"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"Claude Opus 4.6","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; 1M total-token cap; no context compaction","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2","source_published_at":"2026-04-16","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"3d77a25c729c969ff20aa876cc6ca4d41ae81e2478a5c3bfeb6dde3cee56498d","result":{"confidence_high":56.642626,"confidence_low":52.742952,"sample_count":2500,"score":54.7,"score_display":"54.7","source_stated_rank":null},"run_fingerprint":"bf2985f6c9eee95c5399a46abb7838e2ba15a8a2ce70475a1d74efb540c50333","source":{"content_hash":"a3416021494c6882904f6a123b4a9d3761d9596b999d8356f9aaf644fcf7aa14","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf","id":"anthropic-opus-4-7-card","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.8.1, p.196","name":"Anthropic · Claude Opus 4.7 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c93fdc26da7ffe52c4cf6b50","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:27:26.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:27:26.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6ec5b294a99f1299e348780085f279761b94ff52edf0b8191054325adc96d8ee","metric_details":{"best_score_across_scorers":"0.517","model_release_date":"2026-04-16","stderr":1.5810153729833274},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"2c4rvmf5N7QXTugrFLMZ83","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2c4rvmf5N7QXTugrFLMZ83.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/2c4rvmf5N7QXTugrFLMZ83.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"f71750a6cf185b750ae565540c105f363a1e1c1a10d96ca8f6af6b68524e1165","result":{"confidence_high":54.798790131047326,"confidence_low":48.60120986895268,"sample_count":1000,"score":51.7,"score_display":"51.7","source_stated_rank":23},"run_fingerprint":"22f155a15b5e0beadfec45e7e9ea8d018a3e29729e2ef5ec2ce90d811ddacf26","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_509db2bd77140fb589d8a000","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b38be37bcc498f66ce6ce807f89b739e387c9c3a3a0b26acd0c149c1916a8d22","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.365129,"date_is_proxy":true,"latency_seconds":202.52,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.545},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","few_shot_accuracy":90.909,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-7","zero_shot_accuracy":89.394},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"ab0d68f1147cb9fddfb6c4beaa55cbe0745c92fcbbda15e24f2a4c53d71338a0","result":{"confidence_high":93.1802,"confidence_low":87.1238,"sample_count":396,"score":90.152,"score_display":"90.15","source_stated_rank":25},"run_fingerprint":"29ca2db320f8bdb7fa6e259ece164f85d6e474d72a22265fd24df49e8ded6f7f","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:19:01.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:19:01.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9ea04b9cdc0b5789e68aa8414f547efdce77edafc2646fda20f118f6b39f27d5","metric_details":{"best_score_across_scorers":"0.8636363636363636","model_release_date":"2026-04-16","stderr":2.445015597318985},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Ju7pUkkwYVj6ugi9Ri5Jjp","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJu7pUkkwYVj6ugi9Ri5Jjp.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Ju7pUkkwYVj6ugi9Ri5Jjp.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"463c0b4d3d09fe92bdb89081190d6a6ffd864d2b20b98b2408f85303a4741fca","result":{"confidence_high":91.15586693438158,"confidence_low":81.57140579289114,"sample_count":null,"score":86.36363636363636,"score_display":"86.4","source_stated_rank":76},"run_fingerprint":"1fc62b7b5b2103b8aaf9c30c011d37071a5ef844c3d9e9c41cb4e776870efa84","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":170,"at":"2026-04-17T16:16:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-17T16:16:45.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4090090ef6e51b2d1750a0695d99486bc336b556d4f7c8493547fd00688b3896","metric_details":{"best_score_across_scorers":"0.9015151515151515","model_release_date":"2026-04-16","stderr":1.8464275887293338},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"VJezHrJ5YPCbpui8pZgcJg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVJezHrJ5YPCbpui8pZgcJg.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/VJezHrJ5YPCbpui8pZgcJg.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"8bc70011039c37bfad80d8b1785cc73d0e37e4f9a82de27d085ecdf73df9b0ee","result":{"confidence_high":93.77051322542465,"confidence_low":86.53251707760566,"sample_count":null,"score":90.15151515151516,"score_display":"90.2","source_stated_rank":44},"run_fingerprint":"37a5701de6c2920ff2a97bf238537f203be6f190b44dd0410165441b0399d15b","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_509db2bd77140fb589d8a000","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"18f9f2a96e9293005f73c84002aead9d72af122d66a0820eeac861596bb83c54","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.061711,"date_is_proxy":true,"latency_seconds":187.83,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.305},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-7"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f58221e5e8ac50c4ef09eb203abc7596277c1587f1b62c43bc62ebb5a42d692d","result":{"confidence_high":90.4688,"confidence_low":89.27319999999999,"sample_count":null,"score":89.871,"score_display":"89.9","source_stated_rank":8},"run_fingerprint":"c89f0c19f80d85c4052ac9015404ccc02ca9d727ca0467defe25e98f4243d998","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:19:46.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:19:46.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"314fdbfdacc2daf4f99565da835e1293bdc483b36d638ef7e2118c5aed7a3afd","metric_details":{"best_score_across_scorers":"0.8666666666666667","model_release_date":"2026-04-16","stderr":5.124707431905383},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"T4y4iGHMqA8tXCdBKr5GjS","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FT4y4iGHMqA8tXCdBKr5GjS.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/T4y4iGHMqA8tXCdBKr5GjS.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"783bb1d83e7d8cebef6c7c4b8de5e536862f4da779b369aefb639a940dfaab18","result":{"confidence_high":96.71109323320123,"confidence_low":76.62224010013212,"sample_count":45,"score":86.66666666666667,"score_display":"86.7","source_stated_rank":82},"run_fingerprint":"dae2695d52b739eb2c9b022493ee4d68feba490b1e3f63b930a9ba8088dca2d5","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":170,"at":"2026-04-17T18:26:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-17T18:26:45.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"282a9689aa1853e9864e7c667075ccb6edbc01a9be9809077338dba48d1039c7","metric_details":{"best_score_across_scorers":"0.978","model_release_date":"2026-04-16","stderr":2.1999999999999997},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jyCZnpXG2bh9G43X8pT75m","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjyCZnpXG2bh9G43X8pT75m.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/jyCZnpXG2bh9G43X8pT75m.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e6af5e36206c05b0103f5f4295cfe61902eade74fa1965cf9edba61d76c914ea","result":{"confidence_high":100.0,"confidence_low":93.488,"sample_count":45,"score":97.8,"score_display":"97.8","source_stated_rank":24},"run_fingerprint":"034017ea34bbc3e87ecf6181cc6c8a0a1efc6454da59cb97dbd8fed4c28a68ff","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_509db2bd77140fb589d8a000","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T21:41:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T21:41:54.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"2587e6f984e6dd04275cb8a657d6b8c3c24d6e10ec5d735a4ebe9d999d615304","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.7017543859649122","leaderboard_private_problem_count":285,"model_release_date":"2026-04-16","public_example_count":10,"stderr":2.7146911721220195},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"ZkNQ7GER7Mc9FSPmpUzgKH","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZkNQ7GER7Mc9FSPmpUzgKH.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZkNQ7GER7Mc9FSPmpUzgKH.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 4.7 (max)","thinking":null,"upstream_model_id":"claude-opus-4-7_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"2af8f172defdd8b9838fc070e6004727cd471180369f89d718b207d85964416e","result":{"confidence_high":75.49623329385038,"confidence_low":64.85464389913206,"sample_count":285,"score":70.17543859649122,"score_display":"70.2","source_stated_rank":24},"run_fingerprint":"578eb6bbc295674b49ee51257d447bfa32756ea1b312a88a6698598bead16525","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"465926f6042a691f7e894cd8e86cb29ef70e0b5d3f8309d2dbdb3de0ae009739","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.7 (Max)","source_effort":null,"source_model_version":"claude-opus-4-7_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"9600efb6cd1778acbe3b03291aa2b77de82c5d3cda92d4d59fffac24b028bba7","result":{"confidence_high":79.96477,"confidence_low":71.149781,"sample_count":360,"score":75.83,"score_display":"75.8","source_stated_rank":47},"run_fingerprint":"b9b2624a3c028325ada1f1f0b82cc2f107635862887b92a3b270040e2b152d97","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a44d69b5fe2d2e1d0607ae5b5cc479572cfbfa5e4d3f26735203b68cbf7b37b6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"7.43","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7 (Max)","source_effort":null,"source_model_version":"claude-opus-4-7_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"04dcb0953d60b699dc013d5c905cf1aba8abab1c59ae3af50b757986d137cc27","result":{"confidence_high":79.936922,"confidence_low":71.118263,"sample_count":360,"score":75.8,"score_display":"75.8","source_stated_rank":48},"run_fingerprint":"03b95f4a75c8c1482077d0882e02da2e84f96ecb6ee31c10ce51971bb65b3e94","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d2452a13c0cae6c69a9189a566fa115730b19a11da8aa8c3f7ddf568f59b16ac","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.7 (High)","source_effort":null,"source_model_version":"claude-opus-4-7_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"7c0954a176fc6ad1d5d48570ba355e60233e931679f116887727d0e7493fbd1e","result":{"confidence_high":72.920399,"confidence_low":63.352528,"sample_count":360,"score":68.33,"score_display":"68.3","source_stated_rank":61},"run_fingerprint":"5b23aca61fd71632fc7174909d5edf892050a7f6b0aa150e00d319cb1bc95600","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9d12e86eb0fcb31fefd383963930d7276a830b5b3dee5d314606680e9d4502ed","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"3.17","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7 (High)","source_effort":null,"source_model_version":"claude-opus-4-7_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"cf2fd79f40985167897e95f1df02446450ec74a918761a870777d3b9e22ccd78","result":{"confidence_high":72.891916,"confidence_low":63.321645,"sample_count":360,"score":68.30000000000001,"score_display":"68.3","source_stated_rank":62},"run_fingerprint":"b964578e7ea926eacfb02b26b2263e1c09dff11a1cd629c12b299a2786c82a33","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"114e9a6a110c04c808b87e204e18e21efeecb68b035cc531de0d3d89b814d2c4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"2.96","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.7 (Medium)","source_effort":null,"source_model_version":"claude-opus-4-7_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"375c1c9e9287ffd0423fb508e7722317c8bb2bb02474b7d968ac40e64495f06e","result":{"confidence_high":72.131525,"confidence_low":62.49893,"sample_count":360,"score":67.5,"score_display":"67.5","source_stated_rank":63},"run_fingerprint":"ed79c9577f4f22734f486ee2cd630efd7d2949fd598e9ab267fecb41ec9df437","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_40c5e1e12b9e07616369cb66","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5c3c44be0b54e073578f16037eabca78b64bec6d030ad0ff97e375ae10349d7e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"2.38","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.7 (Low)","source_effort":null,"source_model_version":"claude-opus-4-7_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6ae605985c440736f009e4a023f2831d4e3e8a953a0ca682565dcf4523f5775c","result":{"confidence_high":66.939599,"confidence_low":56.965309,"sample_count":360,"score":62.08,"score_display":"62.1","source_stated_rank":74},"run_fingerprint":"fc1c10516d148af71ab0376b02b6e104e6b84584528048a0b438477a879320de","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6dbe767667055c2acd4c9653","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T21:41:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T21:41:54.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"bf2604edde0c1d873c7f853cd496dc6d2aee88fbdff5c2121df2689065d7a11e","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.3170731707317073","leaderboard_private_problem_count":41,"model_release_date":"2026-04-16","public_example_count":2,"stderr":7.3576112824382225},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"P4LXQ9PyDDziEANMB5jVRd","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FP4LXQ9PyDDziEANMB5jVRd.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/P4LXQ9PyDDziEANMB5jVRd.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 4.7 (max)","thinking":null,"upstream_model_id":"claude-opus-4-7_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"4b2aa72f1aa2a2342d6972ee79194b6b97e00904c2418a36cfcd8df4fd9e170f","result":{"confidence_high":46.12823518674965,"confidence_low":17.286398959591814,"sample_count":41,"score":31.70731707317073,"score_display":"31.7","source_stated_rank":31},"run_fingerprint":"ca2f6c823b00c097aef1ed6545dbcc17bd703496c246d668ace7a5bc320b19b8","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9672c259dbb7dcd20a2a424d4596eef53713f5e160d76c84be3f1b1b79ba0351","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.170461,"date_is_proxy":true,"latency_seconds":70.801,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.845},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-7"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6b1ba6ff6c81b3d289ec19635ca06f808fb698b86053906bb1170f3d7b1400c6","result":{"confidence_high":87.2052,"confidence_low":83.89280000000001,"sample_count":null,"score":85.549,"score_display":"85.5","source_stated_rank":24},"run_fingerprint":"952d152a550f52ae2bfc771e7b6b38e92c7c1af0ab5c69ed904ea55fdfffdab9","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"590395591cc524ebd41574c578fa19b66abcb6c48fe064a24a13edc9b5d7566c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-opus-4-7_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"ebdb1826134f5d76dcddd272e4fbf9800158d6132a12d69d58d186d1f72ec275","result":{"confidence_high":21.566879,"confidence_low":6.334181,"sample_count":71,"score":12.0,"score_display":"12.0","source_stated_rank":87},"run_fingerprint":"2a6a12638514a764008327c692953027ab87186900831d21bcf9290348764f9e","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3c65da6f7363c9fb02d33546ed3445b22e7476d983bc3e4826c01e9c764be694","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7_unknown","source_row_date":null,"upstream_leaderboard_url":"https://lmcouncil.ai/benchmarks"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"6b0b62de4c90e22ffbf27aa09e82ef448940e448951663f30e2ebd7aa76798b7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.7,"score_display":"61.7","source_stated_rank":24},"run_fingerprint":"d467ec140944c32c4f0757537a6576d24e28b0b418ec6783f364d7a8102c63b9","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_15740d2b88537909e84c3dc7","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:19:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:19:09.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a16cca1aa5eb16d04cb88704f4ab66692023d2baa70d1b6d15e34b2d078ac93e","metric_details":{"best_score_across_scorers":"0.07","model_release_date":"2026-04-16","stderr":2.564323999762428},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hrRSYo5XSFrcs9fADb5Hox","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhrRSYo5XSFrcs9fADb5Hox.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/hrRSYo5XSFrcs9fADb5Hox.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"922f84395424013e2667063ba493ecd9f833a1d105c0c5ef313ed3145133e426","result":{"confidence_high":12.026075039534359,"confidence_low":1.9739249604656424,"sample_count":100,"score":7.000000000000001,"score_display":"7.0","source_stated_rank":148},"run_fingerprint":"a9e6cf04fd61b31992e3c4df154d5db4d869153045507c244154483f0ff89f64","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":167,"at":"2026-04-20T16:56:00.070Z","basis":"evaluation_started_at","evaluated_at":"2026-04-20T16:56:00.070Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8b9463cd6a9c88f3e0adea610a1cf72fde0214bbbb30288c6a3799cef58cc69b","metric_details":{"best_score_across_scorers":"0.3","model_release_date":"2026-04-16","stderr":4.6},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"opus-4-7-chess-puzzles-prelim-lower-bound","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1c62a5a5f60f3f196df62978c60d2528dffb1d54f8ca2ef5ff80b603f1a281f1","result":{"confidence_high":39.016,"confidence_low":20.984,"sample_count":100,"score":30.0,"score_display":"30.0","source_stated_rank":51},"run_fingerprint":"754683bb7214b7f6947d91deaa278121e24b4b0f47551da40dd0e66b376228fe","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_509db2bd77140fb589d8a000","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-14T23:34:59.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-14T23:34:59.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"94174b4d53dafb62054f3feb6ccbff8e514b54633dedea6046530835dd48303c","metric_details":{"best_score_across_scorers":"0.2","model_release_date":"2026-04-16","stderr":4.020151261036848},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"T7KJtht8Fesh35M5ayMPnE","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FT7KJtht8Fesh35M5ayMPnE.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/T7KJtht8Fesh35M5ayMPnE.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"88a243cc3dd35feb35162467c053b69b63b98bbc98708b6c1c4441ebe97a82e9","result":{"confidence_high":27.879496471632223,"confidence_low":12.120503528367777,"sample_count":100,"score":20.0,"score_display":"20.0","source_stated_rank":87},"run_fingerprint":"dcbf2144e4270373f0a17a2943b783eb6ce37b78a02dbf380477c44acc8125ad","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6dbe767667055c2acd4c9653","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1e8ee34d5828a4539e2184d7ebdd767c7ae9fef0f984d44d844bfb2800ec0035","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.58","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":""},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7 (Max)","source_effort":null,"source_model_version":"claude-opus-4-7_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"9e60576dccc5cb95fc28b9550b2148d78cc1e9210cff2b06fa0049c9379b7c89","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.0,"score_display":"92.0","source_stated_rank":59},"run_fingerprint":"cfe202793edb1cc51038709aca449468ca7478e6aca41e9312240544fd9554ff","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1e0102f187802c9b83f81b000d27f16836ba6a8ae9b8666e4c92a981f7cffd4c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.7 (Max)","source_effort":null,"source_model_version":"claude-opus-4-7_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3aa15245f9eab88ea215fe2004bb36107e387841392a0ca6634f7d3ec61c1eef","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.0,"score_display":"92.0","source_stated_rank":59},"run_fingerprint":"698667ca66bf4a13bf94aaaf6faa33f35d3a8fb802ab220d58969117b8fa14aa","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ad50c6946799eb15443fb1528dbb4e953ce3eb60d66de587239b59f398a889c1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.41","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":""},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7 (High)","source_effort":null,"source_model_version":"claude-opus-4-7_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3baa9f3fd37d32af704cf2c97e06cf2f3a758c155a4f183d9374d3de4580c813","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.5,"score_display":"93.5","source_stated_rank":48},"run_fingerprint":"b0aaf4fed2bbea41ed0788a7abf931ef8043ec0549fef07fee69119b6791ba14","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b26e6f800cda345fc66462cf4864dad53dbdce5aee4fff68b026b4530ec9e573","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.7 (High)","source_effort":null,"source_model_version":"claude-opus-4-7_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"aff171893a3c3da0ff60d86ebdda3feccbbb31e18d0b21a7c9d07e50de503804","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.5,"score_display":"93.5","source_stated_rank":48},"run_fingerprint":"ec71e0332166dc5d6527788426250d83113d88c3f27698463b9c4678d11f6060","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"52817734a7eb74dcbf4ac3ec6e03c634620626522b758c32381a407d3b8accc3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.04","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.7 (Medium)","source_effort":null,"source_model_version":"claude-opus-4-7_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d723cd4dd0d0adc382948094339ea48415a98d083dc85a0300d3dbce71432dd8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.0,"score_display":"91.0","source_stated_rank":67},"run_fingerprint":"3895a21f984a29e80a712cbcf48d1218721a8352b39835a45ca5c7eac10a0e18","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_40c5e1e12b9e07616369cb66","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0217f9e8706e79c6d4928a189f25f0c5ec88c0f1c3e2bb683f646c595fdcd06b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.76","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.7 (Low)","source_effort":null,"source_model_version":"claude-opus-4-7_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6291afa103021d8b7a2ec98e29f5841058fd874f341a71a1a5c7f6a6223a5229","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.0,"score_display":"91.0","source_stated_rank":67},"run_fingerprint":"c62d6208279d3c4e26b64c533e8bee9d33b240885e995b19b2935cd2f25c3b2f","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6dbe767667055c2acd4c9653","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c0509cd6f5b5d6cee523d4bb43e5afaec0659fdb089e7423e417cdea220b7c34","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.423427,"date_is_proxy":true,"latency_seconds":441.988,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.72},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-7"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"682f4e4ba6303a01484eb88b3274ffac8f9e8bd97642141b29a0c36b76cc7ce6","result":{"confidence_high":85.3712,"confidence_low":78.6288,"sample_count":500,"score":82.0,"score_display":"82.0","source_stated_rank":21},"run_fingerprint":"0ab68f4b02236a64ec5c88df90cfbd9ed4835e52ae0398f1ed6082f04b4b3431","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":168,"at":"2026-04-20T13:16:09.316Z","basis":"evaluation_started_at","evaluated_at":"2026-04-20T13:16:09.316Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0d49e8ede3fd2df77a5227af7ee85461167f0dde171f18c495adda7ce68bd93d","metric_details":{"best_score_across_scorers":"0.8347107438016529","model_release_date":"2026-04-16","stderr":1.6901169179529603},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"nCmGWWMBip2s9AVEZNYXDX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnCmGWWMBip2s9AVEZNYXDX.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/nCmGWWMBip2s9AVEZNYXDX.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"0bb0388aa485f17123ac30c0bb187ab6f127c7ed177b1b5df1e9e057d41cd92d","result":{"confidence_high":86.7837035393531,"confidence_low":80.15844522097748,"sample_count":484,"score":83.47107438016529,"score_display":"83.5","source_stated_rank":1},"run_fingerprint":"ad92fcb7bc58e167a0a5b4fdd0dafa3a158ad0d108d498b1c95e33ba7a951bf6","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ed49ef6cb40ad079db5a0835","split_or_window":"verified-500","unit":"percent","version":"Anthropic 500-task evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"ca258432fb2356d75a16b041eeeeeb45dae0048aebd0922c46198d5ebb6a27c5","metric_details":{"comparison_warning":"Vendor run over 500 tasks; the agent and tool protocol is not fully disclosed.","comparison_warning_code":"swe-vendor-500-protocol-incomplete","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.2, p.192"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"swe-vendor-500-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; default sampling; five-trial average","question_count":500,"reasoning_effort":"max","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2","source_published_at":"2026-04-16","tools":"not reported"},"run_count":5,"scaffold":"unspecified agentic harness","tools_allowed":"not reported"},"protocol_fingerprint":"ed5a57a6d5f71c1a59d81da5bfcc604e7ded280f45dfc2a7429ba71f787e3277","result":{"confidence_high":90.205434,"confidence_low":84.421195,"sample_count":500,"score":87.6,"score_display":"87.6","source_stated_rank":null},"run_fingerprint":"2a11e18d4637c50c9d17b97b8f9546e1c129a2532aefce9b8faa277bb69b7bcf","source":{"content_hash":"a3416021494c6882904f6a123b4a9d3761d9596b999d8356f9aaf644fcf7aa14","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf","id":"anthropic-opus-4-7-card","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.2, p.192","name":"Anthropic · Claude Opus 4.7 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c93fdc26da7ffe52c4cf6b50","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_332cbf0e979daa9e173de9f4","split_or_window":"2026-05-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-29T23:53:02Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4df263ff2ccd91dcde8f47b1aef9c32231f8c94af170d6d99bbc9176f3b84c80","metric_details":{"comparison_warning":"Official post-release SWE-rebench window; use only for same-window comparisons and never mix it into the canonical rolling-window rank.","comparison_warning_code":"swe-rebench-post-release-window","model_release_date":"2026-04-16","pass_at_5":57.14285714285714,"potential_contamination":false,"sem":4.259177099999599},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-post-release-window","comparison_warning_code":"swe-rebench-post-release-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","supplements_range_key":"1772323200000:1778803200000","upstream_model_id":"Claude Opus 4.7-high__tools","window_from":"2026-05-01","window_status":"supplemental-post-release","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"56db8c9f4e6d312b6b00ad167c406430100335479f46b025cf03f8521d17936d","result":{"confidence_high":51.205129973142064,"confidence_low":34.50915574114363,"sample_count":null,"score":42.85714285714285,"score_display":"42.86","source_stated_rank":null},"run_fingerprint":"849ff548dfdb194ba82e9190319713ff80cf1c4cb070192eeb282eb8790e95eb","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-post-release-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"41e07df6393b40cb2d9de4fb99e164dd440f0378b7730ef8debbaec01f499df8","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-04-16","pass_at_5":66.36363636363637,"potential_contamination":true,"sem":1.4545454545454541},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","potential_contamination"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"Claude Opus 4.7-high__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"760b451c6ed4ecd9fc2438e6c69c6fb389c06c7d39100d7b8d503da895abe579","result":{"confidence_high":55.94181818181818,"confidence_low":50.239999999999995,"sample_count":null,"score":53.090909090909086,"score_display":"53.09","source_stated_rank":null},"run_fingerprint":"af0265a00d59296fdcae917e9d3ae9ce9dc2112b3bc8cbfd4c6d43de4f40ad1f","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_6323abbd504101af2cdaac58","split_or_window":"vendor-public-own-harness","unit":"percent","version":"SWE-bench Pro · Anthropic provider run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"35f0d042c59cabaef8b03609f4f4722aea42be364bfa88f19faf706e230b10a9","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.2, SWE-bench Pro capability evaluation"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; default sampling; five-trial average; exact public-task revision not restated","reasoning_effort":"max","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2","source_published_at":"2026-04-16","tools":"software-repository shell and code-editing tools"},"run_count":5,"scaffold":"Anthropic agentic harness","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"623d17d2d37c45c76a04ed88c374201cbc38e405ccaa2ce37c341a5fed0c0a30","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.3,"score_display":"64.3","source_stated_rank":null},"run_fingerprint":"8f73909ba331a80872edce6bd5a338dc0b4d4ac22f558956f55f933556dddcbe","source":{"content_hash":"a3416021494c6882904f6a123b4a9d3761d9596b999d8356f9aaf644fcf7aa14","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf","id":"anthropic-opus-4-7-card","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.2, SWE-bench Pro capability evaluation","name":"Anthropic · Claude Opus 4.7 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c93fdc26da7ffe52c4cf6b50","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_50545b611e994b43926cee37","split_or_window":"google-single-attempt-comparison","unit":"percent","version":"SWE-bench Pro · Google comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"0f410ac73f5e291694f815c076e837d5c91e9064105a3fa9ce22d53e89602982","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · SWE-Bench Pro (Public)"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google DeepMind provider comparison; single attempt; exact scaffold, task revision and thinking level not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Google comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"f4da5832d81605e31c85f7538ca6eb99e6fdd9f6e21d5a1a7abad17d95b051e9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.3,"score_display":"64.3","source_stated_rank":null},"run_fingerprint":"99612e842591deb33ca9a207ace1b49f604137f03d1c6f5922bac6bde30ac43c","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · SWE-Bench Pro (Public)","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_853d579a7a3eb11e93ab5106","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"6d0c40d322fcd03d5bb675af4762bfc48a57bc998ead669e92a803024919b563","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"11.02","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"96","tokens_per_task":"62989","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.7 Max","source_effort":"Max","source_model_version":"claude-opus-4-7_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"ca32bda5f37bae32e92a511b0c956ebbd63010638a5d403c2a8d9b19aaa26745","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.8,"score_display":"64.8","source_stated_rank":20},"run_fingerprint":"4ca085bf2ebf07cae240699ef0bef5ef25d9ef2d5908df37dd3f564f66a8c827","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"07290c4c69a1d0339b7cb8aa63896b7bf449c7ebf677d5c3e109f3ea59e4e6d4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"7.11","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"72","tokens_per_task":"43942","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.7 Extra High","source_effort":"Extra High","source_model_version":"claude-opus-4-7_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"466d167c316219f0750e4e2eeb64d6b08ae04a209734e141c64bab4d17428106","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.6,"score_display":"61.6","source_stated_rank":27},"run_fingerprint":"394813f2b3de8662ace7370eb3405162ce2a8341a3d2c43fb9bbd089dfb3347d","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_509db2bd77140fb589d8a000","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"8440194451e439592f425365871bb5e3bf05b0e937b5bda3a8eeec5b85fd1d09","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"5.01","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"59","tokens_per_task":"32227","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.7 High","source_effort":"High","source_model_version":"claude-opus-4-7_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"0d2412e8775dde5455e4df31412839b9ab4bec203a1420683cb8ca498f54d3ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.4,"score_display":"59.4","source_stated_rank":35},"run_fingerprint":"a8451a463dd9ee6e1b5bc20ff88e8b4d7f830ddd9c8ef23b2f16129c408b5857","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"9cde393fca1de36dde060b9f09e039b4291ec34974edda80c6199322dd2f14d4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.93","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"41","tokens_per_task":"19193","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.7 Medium","source_effort":"Medium","source_model_version":"claude-opus-4-7_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"3aae9d55ba7671bcdc67d3797aca14e853a3cc935de07d63c367e76a3641ae5a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.7,"score_display":"52.7","source_stated_rank":54},"run_fingerprint":"fb1795694fbdc0a2d7ff67041a4c4c315abdafbf9bdcb8db05e27143e40cb446","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_40c5e1e12b9e07616369cb66","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"fb2231178a0d6cd67a94edf39e086a21514f7cee3584ab4e474f763a1fd96f83","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.87","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"29","tokens_per_task":"13164","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.7 Low","source_effort":"Low","source_model_version":"claude-opus-4-7_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"e1f6db4a2940955850812f74d801382efbf5be1223f25f4488e5cb5f162342fa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.3,"score_display":"48.3","source_stated_rank":65},"run_fingerprint":"52b7d2dd54697dcdc46c6e2b34ee85bb332b397dac5ad623f93208b9f304511e","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_6dbe767667055c2acd4c9653","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":161,"at":"2026-04-27","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-27","status":"stale"},"measurement_id":"a7f2955e83c6d910688187464b4c3f983745eee0873e2ad70ea490897576a764","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7_high","source_row_date":"2026-04-27","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"90fc44bc011dd559e0a60b3e81a8069f34dec6660bf522853493ffc7bb164ea0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.12,"score_display":"44.1","source_stated_rank":2},"run_fingerprint":"ab39918aaad384bf2b34471d7eb8ec3ba42cf6e04b425bd2d994a849a1282a4b","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":161,"at":"2026-04-27","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-27","status":"stale"},"measurement_id":"a7ce1fdbdc0a3c99b8013458f4ff7c0cd51f35c18eaf1a3f9c15c2a74e3ceb90","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"GSO Leaderboard"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7","source_row_date":"2026-04-27","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"f3f0df34fe5f0160d89b3b6e420f6b7cc82a587d71c682d692a107771340c8a7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.12,"score_display":"44.1","source_stated_rank":2},"run_fingerprint":"3da2cb606845b4a05b0e34622289e3470949c9547f2bec87dce847117d6dcb32","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6b47fe6f37aac6a0bb720bee","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"898bc9fa53b24e8c6221b28adae1334c807a972646807bbac355b0e53c840226","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7","source_effort":"max","source_model_version":"claude-opus-4-7_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"cfb3195eafc5e45231888af39f986755ca83f74a702dcb5243a8852e0f4f4942","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":38.54,"score_display":"38.5","source_stated_rank":24},"run_fingerprint":"f2cf52f0ca1fc6a2850c3664cb907358427aaf1d1cbb2719f85f976a321915fd","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d0d905b95fc9889e4e35f5840e3e105ab2dcf42b29f6f2926dee43453a0caa29","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.350587,"date_is_proxy":true,"latency_seconds":461.91,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.006},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-7"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"412aa8f553efe9d4199b9387acf0880905ef76fd20360e00bd85d6c40afe504b","result":{"confidence_high":87.04476,"confidence_low":83.10123999999999,"sample_count":null,"score":85.073,"score_display":"85.1","source_stated_rank":36},"run_fingerprint":"7d9d9127dade29dc5f693b927d91a080e5b07e8a8e990741784ebaeb3cb68e31","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"46ffe45f302d3d6d9c5f86d1413801d6736458f367386135fa3c64a6beb523a4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-opus-4-7_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"5d8db5a2977ad2d5ef6e4b5cca401b12041207b8e5311c696ee76bdd3ca238d7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.51388888888889,"score_display":"54.5","source_stated_rank":49},"run_fingerprint":"7d342a6a448b4461181dc22083703359c591e42bf7beb4f35c05b59ecd4c431b","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0a929c1064502ee1362c54aed95e75747cd8161dabd2449d75ce2d2fdfd1d4fc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.224549","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-4.7-opus-no-thinking","source_effort":null,"source_model_version":"claude-opus-4-7","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"139cf0ed306fb30eb9c7474fb6fdbcba0ad7b6b87896baf0fe2b823b0e87b094","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1323.05,"score_display":"1323.05","source_stated_rank":25},"run_fingerprint":"bdf8e653d477bbef0308ba655e16b046438b42c1f9e19c4de81e40dcf36982ce","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_a98b3983e054c2e804d5eadd","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e3dda0b8d3944bcd5ef85973a4aaea3638643850eea16a84e2bc65d0e9706f64","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":21.407199,"date_is_proxy":true,"latency_seconds":2141.902,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.506},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-7"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"06237c291be38e853f9a063a19f19af74af808339b6a803811979e86c1449099","result":{"confidence_high":79.83476,"confidence_low":62.17124,"sample_count":null,"score":71.003,"score_display":"71.0","source_stated_rank":33},"run_fingerprint":"00b1ad82f1415f14d1eaa159f20e6741e634c4351b14e7b54f1c7ee941f009fa","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef64a20034a4e320aa729d4880bb496c0c763d310cf2c457af7233e58f56283e","metric_details":{"license":"Proprietary","variance":8.619226980219103},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-4-7-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ef85de20200d4c0213d6f270220070078ae8e34bb4c8b5bea82237d7059a765b","result":{"confidence_high":1563.0967502583421,"confidence_low":1551.5884218809056,"sample_count":17982,"score":1557.3425860696238,"score_display":"1557","source_stated_rank":34},"run_fingerprint":"e7cdf7de6485a3efc92df4536e58f0f7b685e72fc9c0b88dd3a360f1c449b15a","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c6e91028968db246a4b96886766d020a326be01ead08c6ae9681b5e886b9ed78","metric_details":{"license":"Proprietary","variance":8.259475062057996},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-4-7","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"98e3615540e5f010a15ca65d4b9e1754694c8af43fd68a384caabf6cbb1ee7c6","result":{"confidence_high":1560.8911385302206,"confidence_low":1549.6255389755684,"sample_count":18174,"score":1555.2583387528944,"score_display":"1555","source_stated_rank":36},"run_fingerprint":"e907e8f2bc1f28c96b8b6900a4643643e5c21236d4573ebbf7e85c95b8d897a8","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c4e5f2a70986767754508768","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ec80b940aec80b7cb80ed34f06b3abb759e22eaf2ac44ac1ea219662199a5f68","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"101e8dbf17af02bea550145556f68fa8071affc10445a978f1b7ffa0571e6ced","result":{"confidence_high":1561.2,"confidence_low":1549.23,"sample_count":16387,"score":1555.22,"score_display":"1555.22","source_stated_rank":32},"run_fingerprint":"ffb23a76df72375300dfd65c3c89bd58dfe81f831915c7b036396cdf266c9927","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8feff00687108a3c679e891c585cb2aad825374f4cfdf5766700ec52b3e8aa5f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"d56358910b5e895ade3ff76c106e00387f196f23c657178487c5554cbc53f611","result":{"confidence_high":1563.34,"confidence_low":1551.4,"sample_count":15914,"score":1557.37,"score_display":"1557.37","source_stated_rank":30},"run_fingerprint":"71e356de11427d7c431070f3bc7e2e8fd203e36288d0b1eb732c4f0b818d8186","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_73c793c15930a2d6cd2a8430","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e36acb6a781c4e4bac64bb90a0c5440f702e1d637c9e9da70027b5c044724976","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"4593f4d96ce77b054e15c71a4db4f1d6c1129080543771049d94aca1ff40cb3f","result":{"confidence_high":1576.37,"confidence_low":1557.33,"sample_count":5270,"score":1566.85,"score_display":"1566.85","source_stated_rank":28},"run_fingerprint":"0a20ab1caa14aaa97ac5577a031ac642483b4818fe5f56680b7287c2dfd41c06","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_73c793c15930a2d6cd2a8430","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"mirrorcode","metric":"mean_score","name":"MirrorCode","release_id":"release_07bf24088c0805f7b6df7a88","split_or_window":"15-programs-30-language-tasks","unit":"percent","version":"ML +Private 2L"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":54,"at":"2026-08-12T11:43:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-12T11:43:31.000Z","first_observed_at":"2026-08-28T19:57:34Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2a84bb68c1c595e91a1797569385003e714055e694921917b90c6a7d0b802008","metric_details":{"best_score_across_scorers":"0.3111111111111111","model_release_date":"2026-04-16","stderr":9.036473223803354},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mirrorcode-ml-2l-claude-opus-4-7","log_viewer":"","logs":""},"run_count":3,"scaffold":"Epoch AI Inspect ReAct agent","tools_allowed":"shell, text editor and visible-test evaluator; no internet"},"protocol_fingerprint":"10132dca49c37c2b0b2fa9a13ab931265d36626de6efa155ea0e727836011035","result":{"confidence_high":48.82259862976568,"confidence_low":13.399623592456539,"sample_count":30,"score":31.11111111111111,"score_display":"31.1","source_stated_rank":5},"run_fingerprint":"56b04519da45e7a65970895b5b8d6846da538335cc21075b49a8e03e1949efe4","source":{"content_hash":"3e212e97e03d0343e676ecc7ff046ee7b94d219ba7d863364634b7361dc89b4b","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://epoch.ai/benchmarks/mirrorcode","id":"epoch-mirrorcode","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · MirrorCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/mirrorcode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"310cdc15b20626814b82d5ec55b6f16c91875531e3b5b67d2026d05ca615c08e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"b8796d5baa9303df1dafed6e88dba11f1571c9db52d952b1f14c7739aa7ee416","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.45,"score_display":"75.5","source_stated_rank":28},"run_fingerprint":"9b5fde4c594eff3ce9f32077c790d84a6ed8469bd7fbfe00637ac630a77cae26","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"125eaac1b19611814ba93910492f3d66d9f8b6f1488954f61d6e9999db452c13","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"4c6701aa802f5a5a56310fa15c67767e9f6b7d0fa45e68f0d60816dd118750a9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.44,"score_display":"76.4","source_stated_rank":24},"run_fingerprint":"09d41c7944d8150953771870e7757a9f73988dbef62e39681f81af68a2fab964","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3367977557dcc845f65cf9060cfca589e73e6cae32c1eec8537257fb0b6b2413","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"8c97ae4f7769787ad9d3e784f49b2e095f4e9786c8f67a9d575e8f1d21af8184","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.4,"score_display":"76.4","source_stated_rank":25},"run_fingerprint":"a20268b2d11ec30d063d2c0e2619e8050354e53b2d48a6f946caae96bf7b88a4","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_12563e08b16dad5441caefb9","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"68254b70deb4d76b57753c3ab1414f58a164dec0a63c873d81da4db2cd028e97","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-7_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"86eb16e2e890603c25f5a2031c9af4907891b52d44470dc2bfe801cf48c2acd1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.4,"score_display":"76.4","source_stated_rank":25},"run_fingerprint":"dd830ddd37f3cc13ea860e574d14d01d1231edea7fa0318cd0240252cd52dd39","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_12563e08b16dad5441caefb9","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":172,"at":"2026-04-16","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"3f339d76b2b016348ea16f0f6d66e2f1f7eef8ead2313e629035943d7608e2e5","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"standard_error_points":2.1123213002,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.7","source_effort":null,"source_model_version":"claude-opus-4-7_unknown","source_row_date":"2026-04-16","source_scaffold":"WOZCODE","upstream_leaderboard_url":null},"run_count":null,"scaffold":"WOZCODE","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"18cf30f6667b086c370e26d693542c7a22cbc2d6a1d4542dadbfc0b3e9ea83f3","result":{"confidence_high":84.36486884949201,"confidence_low":76.084569352708,"sample_count":89,"score":80.2247191011,"score_display":"80.2","source_stated_rank":6},"run_fingerprint":"c25d254a7520b0720c826b6b23112d1b7790a550e702e841d4ccedf4dc831e31","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c480aaa87dc483755822700","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7f23a6f5257a148c18afbba99c180d0930793c8482242d2b61b3429b674065c1","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.977087,"date_is_proxy":true,"latency_seconds":816.587,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.297},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-7"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"42e45136068ae9d3a64fe8b5d2e87e86f57968fd6f369a31f5cd0eedc8e66d71","result":{"confidence_high":71.08112,"confidence_low":65.99688,"sample_count":null,"score":68.539,"score_display":"68.5","source_stated_rank":31},"run_fingerprint":"af007d323bc45391060bab637ffb9a39265243f2577f05ca80ddf8893a9b1705","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_1c905f9be1ce4940b60cc62b","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_39c649809acb869ddd02ba8c","split_or_window":"terminus-2-provider-comparison","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"3cb27fe4441fda2872ad2bac8e391a575cda098f1ee0418d82b2c5d41df3ef2f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · Terminal-Bench 2.1"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider comparison; pass@1; thinking level not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"2c2dadcb037ba9267c46dbbb00984e8a635597ac6a0dc521f04cb58942f74299","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.1,"score_display":"66.1","source_stated_rank":null},"run_fingerprint":"5a13982d094f8697121681e4436627f76166d703b511e6dc0d476b9589bcb693","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · Terminal-Bench 2.1","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_853d579a7a3eb11e93ab5106","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":74,"at":"2026-07-23","basis":"evaluation_at","evaluated_at":"2026-07-23","first_observed_at":"2026-08-28T07:09:05Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"03467d19d59b2e01e2cb4c2288f7d5d6f68b5795d62a84aace83d01198c031c0","metric_details":{"cost":null,"pass_2":31.099656357388323,"pass_3":27.061855670103093,"pass_4":24.742268041237114},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"claude-opus-4-7_sierra_2026-05-05","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"ce4e31865bc9890925719bee70ecfc72a33c3c49e73aeb5cd7164f1ed47c7997","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.20618556701031,"score_display":"40.2","source_stated_rank":null},"run_fingerprint":"9566ee812187b7ab7d91a38f015287da3dd6ddd71a7c15684dd9a9d096522e86","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":179,"at":"2026-04-08T17:02:16Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-08T17:02:16Z","status":"stale"},"measurement_id":"4b23a80b99edad04692bab445e0b933a3a3f1f80680d46649a096f7e0a3bfb34","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.5,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=claude-opus-4-7 (max); createdAt=2026-04-08T17:02:16Z","source_row_created_at":"2026-04-08T17:02:16Z","task_pass_coverage_threshold_percent":75},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"claude-opus-4-7 (max)","source_reasoning_label":"max","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"79d9b3839152ddc549f66b5503c9ca73641cd9d4eb056574af3136c0c638fd19","result":{"confidence_high":81.6,"confidence_low":76.6,"sample_count":1000,"score":79.1,"score_display":"79.1","source_stated_rank":3},"run_fingerprint":"97ccc1633e3dc2c391b8c31e8eef5122038f15defe2378a718600a0f4ebae777","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=claude-opus-4-7 (max); createdAt=2026-04-08T17:02:16Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"ae76e7c3d72abd479e34a03252e1ace9637f404fd18702c3e774542b855620fb","metric_details":{"expected_trials_per_mode":261,"normalized_gain":31.9,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":18.2},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"claude-code","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"claude-code","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"3b4651a644f962cc455e244c75c19a44ac9a846ec9096a32e403b17de579282a","result":{"confidence_high":69.8,"confidence_low":52.6,"sample_count":87,"score":61.2,"score_display":"61.2","source_stated_rank":3},"run_fingerprint":"a637cb26046e8e5896dc3aad91f0828a337cc87c9525e2d17db01bd374f5b271","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7135443fd9ef4d3d33bd3e91","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"c84635294bfb182bcf476880c3e2b638ae9843577d0cd0c8d788e57314883d52","metric_details":{"expected_trials_per_mode":261,"normalized_gain":19.1,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":11.1},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":62.3,"confidence_low":43.900000000000006,"sample_count":87,"score":53.1,"score_display":"53.1","source_stated_rank":9},"run_fingerprint":"a3d64a197dedbd38b68b6ec56c2183a9498b96aadf730e66f1e8bb79deaf1fac","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7135443fd9ef4d3d33bd3e91","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"e669b9f7346581573defa7c6f7b57b412938b4cba9356c0bdafeefa950690e99","metric_details":{"expected_trials_per_mode":261,"normalized_gain":31.9,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":18.2},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"claude-code","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"claude-code","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"81f14c5049e718db52e8c57e1c2186e85ae2da7a22cecd1c0668a74f0fa35dda","result":{"confidence_high":52.4,"confidence_low":33.6,"sample_count":87,"score":43.0,"score_display":"43.0","source_stated_rank":4},"run_fingerprint":"f2e5510292c9ede0b16eb5c41f3fa3722a453db4d2966acf2b65fb10ea281a3f","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7135443fd9ef4d3d33bd3e91","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"9e3cf59f4c966ca1b96f23770017becfe3b5197440ea813b33c9d5c58269b916","metric_details":{"expected_trials_per_mode":261,"normalized_gain":19.1,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":11.1},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":50.900000000000006,"confidence_low":33.3,"sample_count":87,"score":42.1,"score_display":"42.1","source_stated_rank":5},"run_fingerprint":"10e2ac7666fdbe8a099c25dc29de09f94f4f18de2221d30c0981620aec7d2680","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7135443fd9ef4d3d33bd3e91","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2bb30e8cf5c195aa85741f4881fd4ae5bfaa5d10557d6c98a5679746c24af4fe","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.631","mean_standard_error":"4.8","model_release_date":"2026-04-16","notes":null,"standard_error_points":540.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.7","source_effort":null,"source_model_version":"claude-opus-4-7_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"9e1f4cd70ed8cf661f298fb62d369da1cfb2864ceb96de61fc2a379cd4e0623e","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":49.2,"score_display":"49.2","source_stated_rank":24},"run_fingerprint":"5051d3f9f1cfa11e523bb7383f87c6fb6ed4343052ce53ae7c5dd95e4265a214","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"appworld","metric":"task_goal_completion","name":"AppWorld","release_id":"release_3fa145932216c0afcdd9dae6","split_or_window":"test-challenge-all","unit":"percent","version":"official-c1f56015cf7c"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02","basis":"evaluation_at","evaluated_at":"2026-09-02","first_observed_at":"2026-09-02T21:28:04Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cf02cbfb2563fa4f2bbca89b8d80a86dd0c02305f14d98aaad6b013623d36369","metric_details":{"interactions":22.15,"normal_task_goal_completion":88.7,"scenario_goal_completion":77.0},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"leaderboard_row_id":"23715209-d5c9ade3","method":{"name":"LARA MAS","tooltip":"Explorer writes an app-tagged plan; per-app specialist executors run a ReAct loop; single attempt per task"},"repository_revision":"c1f56015cf7c3441ff1933f5bfbec879798d7bbe","source_model_label":{"name":"claude-opus-4-7","tooltip":"claude-opus-4-7 for all agents (explorer, executor, supervisor)"},"submission_url":"https://github.com/Omer212111/LARA"},"run_count":null,"scaffold":"LARA MAS","tools_allowed":"AppWorld APIs"},"protocol_fingerprint":"a224489dfbe69501c48902cbd5c7303930991bca8273c1a7269f09203a141099","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.6,"score_display":"85.6","source_stated_rank":null},"run_fingerprint":"efd7c4125bb761854b420db11628d2ffb03e05c078f3c082ff47ad18d86373be","source":{"content_hash":"d44080bb01254150fab30f845d22262a28ee7c90f7850b98ce0ab235ea043f0f","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-02T21:28:04Z","homepage_url":"https://appworld.dev/leaderboard","id":"appworld","last_fetched_at":"2026-10-05T12:33:29Z","license":"Aggregate server-evaluated scores redistributable with attribution (2026-09-02)","locator":null,"name":"AppWorld official leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/StonyBrookNLP/appworld-leaderboard/c1f56015cf7c3441ff1933f5bfbec879798d7bbe/experiments/outputs/_leaderboard.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_82a761bb656269ff1042ddf1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"appworld","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"66c68e7fb54735e8499f801cfd209452310e8f751bc97a712171850260380459","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7","source_effort":null,"source_model_version":"claude-opus-4-7_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"7e827b07d7108076c5aaa4751be97125bb1c3766c6300a4d9650a45587bcded4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":10936.763333333334,"score_display":"10936.76","source_stated_rank":4},"run_fingerprint":"54632569d394dfe9233d4cd6967abe1c49c545100dd94d5996028daeebe8f3c8","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5560a7522edc54c726d81374","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e8c36fc1a04a21c4d0e343b693e72bd5c7311787b47155f495bb48c723e7107d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-16","reported_confidence_interval":"-21 / +21","source_locator":"Leaderboard models table · Claude Opus 4.7 (Adaptive Reasoning, Max Effort) · gdpval"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-opus-4-7","upstream_model_label":"Claude Opus 4.7 (Adaptive Reasoning, Max Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"8f60a26f6712d4c0d98eec2c811c9e48b0d19bd3c62a2ba2934c0fe2ffb3fb93","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1338.42,"score_display":"1338","source_stated_rank":null},"run_fingerprint":"e0ac461bf3f052b2912767399f4889def3a58ab56642233529322a4dcb520892","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Opus 4.7 (Adaptive Reasoning, Max Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"83ca3df0cf70271c2516f53b87300615935dd09e2e826cd0c2574c6adf32ef3f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7 (max)","source_effort":"max","source_model_version":"claude-opus-4-7_max","source_row_date":null,"source_scaffold":"batched tool","upstream_leaderboard_url":null},"run_count":null,"scaffold":"batched tool","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"8324b7d07a58972148ce8c7a94654e6da9a57c9a8f154331c3e283d10e6c2175","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.2,"score_display":"18.2","source_stated_rank":9},"run_fingerprint":"23dc18802a1c98c978b48ca16b00a3d31a0858c267ff006d9ed8750abb6a850f","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"904be1f8207aa6e75cce437c1a22df014c93407c00f12a12cbf8e69cbe3c9ef4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-16","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-opus-4-7","name":"Claude Opus 4.7","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.7 (max)","source_effort":"max","source_model_version":"claude-opus-4-7_max","source_row_date":null,"source_scaffold":"standard","upstream_leaderboard_url":null},"run_count":null,"scaffold":"standard","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"e23a3fa0d6081825c9a9c7d8854670ac9fbaa504c9c875693ca8d764802ce3e5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.900000000000002,"score_display":"13.9","source_stated_rank":10},"run_fingerprint":"1d190c35d1e06e44d0031ab0585aee1375a3045d28fbeb295982e65bb84d6471","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d21e5b67f44d1fce22bb9f73","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c837228108938f92ab1c5f5172e77a00561207ef3c8004e150bbf552e282ceba","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-06-30"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Sonnet 5; model release: 2026-06-30","source_accessibility":"API access","source_model_name":"Claude Sonnet 5","source_model_release_date":"2026-06-30","source_model_versions":["claude-sonnet-5","claude-sonnet-5_high","claude-sonnet-5_max","claude-sonnet-5_medium","claude-sonnet-5_unknown","claude-sonnet-5_xhigh"],"source_reasoning_efforts":["medium","high","xhigh","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"014ac45c9f7550e1df90b484da7772160510faf2a22027d32ee52f19f9ab8a8e","result":{"confidence_high":158.81,"confidence_low":153.61,"sample_count":null,"score":156.34,"score_display":"156.34","source_stated_rank":23},"run_fingerprint":"0e3bea29be7ccc39ca13d899cb365cbff0cd0a12ff485d962a453d9bfc3d2a97","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6600b24d6b7955f517414625","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"558bacf4e91111f27fde0eb092659d59a3e75a34e13f1000118ecd39a1510c59","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"6442df5e345df22195d2746a8025b16de45b53798837d410d1b890cce35fc66b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":38.357696,"score_display":"38.4","source_stated_rank":null},"run_fingerprint":"04976a279a4bb787d963ea6e878f793f4ed5055bb4d59e7a94508e119b331fc8","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":23,"at":"2026-09-11T17:49:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-11T17:49:02Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"edf541bfdac9286ca34c6fb5fffc65233b8b2b6581cdc6466ffc3b2a37f33bcb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-xhigh","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"effd4ac2cd6f76ec10c1229581c706e837a03a8315905bd30fd0e606877e0ddc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.707237,"score_display":"34.7","source_stated_rank":null},"run_fingerprint":"92df7634b9ed14d996c10d8627a19e1d1ffe54d68cf908f3af035aedf41c2ed9","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":24,"at":"2026-09-10T20:32:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-10T20:32:06Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8acf493c2bd91bb71607cb7af6632f594e904ff43fb1a01bc90fe970ac50d4f1","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-high","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, High Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"57158524af07fce5d023a3b3bb14e8b9bc3fac108520c35b5c3718f0e4ecc79e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.0284,"score_display":"32.0","source_stated_rank":null},"run_fingerprint":"23921d23036575c882b9c674e6e908a7850f486ae4653bf8e7d3e6594d8ce818","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c3c234f007a9939e63c38d951c33577092dd6a97e82c287b1168fd89fa241642","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-medium","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Medium Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"c6581264d21db77283884d61f9e740a1d2590e578c543d576f5ba790391a895f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":28.437647,"score_display":"28.4","source_stated_rank":null},"run_fingerprint":"cf2da13c0b9067835719a74b3f76d03f0f4d8ed851c35d844d19914d6990d5af","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_017911fce805da3d633adae6","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0cf43a575697221cacc0ef5d7ecc7fcfd6321818607daa245ef1df7416e123a7","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-low","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Low Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"97c16807a84c8ba2b5df288a849384c43c57226c223600a58ee1eb7ade5bf9e9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.742614,"score_display":"24.7","source_stated_rank":null},"run_fingerprint":"8728a00c7926a8ffdab65c3a423198a5634d7041cb79dbbf552470aa3fb5cf30","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e672ac0437f08612f7384fba","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eaf829c423800c94f38c2832cb8b8148798db52652f224f3f12abacbaf169317","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-non-reasoning","upstream_model_label":"Claude Sonnet 5 (Non-reasoning, High Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"c5e8afa3210576bfd95f9290c7748baf14402742bf32676d96b3d31d904117c2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":28.937705,"score_display":"28.9","source_stated_rank":null},"run_fingerprint":"b013a910b190a220c6f3a7910fdb1ec2f87583c6c6b5a89592a958467f55d63d","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6bca2a2ee2ca5f4a6d2be994","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"34fe6f489050a5cc68c42c0c373595f20ff76cde1e2a3f7ce2fa9a1ba7872825","metric_details":{"license":"Proprietary","variance":4.6098470102008395},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1466.0276250328372,"confidence_low":1457.6113224100968,"sample_count":45739,"score":1461.819473721467,"score_display":"1462","source_stated_rank":57},"run_fingerprint":"d5e421374b3cf0d0c4aea92d8da5f07e5650505c66bfc44d62337588b79afd91","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"40475d3ebb2e4d574e577ec2478b365de30d5f76f0f35ca8a0e62da0d6e862c8","metric_details":{"category_scores":{"Agentic Coding":59.394000000000005,"Coding":80.68,"Data Analysis":71.74066666666666,"IF":63.85850000000001,"Language":74.97033333333333,"Mathematics":92.94225,"Reasoning":88.69225}},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.03971428571428,"score_display":"76.04","source_stated_rank":33},"run_fingerprint":"5cc06d4d07e1b9963e84cb2bf1419de472b4779d4b07164ae6d9fdc1520a6d82","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"df52f43c4637cc137c0de302e32f045a622d5946f587cc4a36535ba21d3312d2","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Table 8.1, p.115; Figure 8.9.1.A, p.122"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; effort level not stated; 1M total-token cap; no tools; no context compaction","question_count":2500,"reasoning_effort":"reported","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f3369573ec5fa708c2fb1887738ca58a6da3b6f6fdd4087151c44904e7e0c723","result":{"confidence_high":45.15076,"confidence_low":41.270106,"sample_count":2500,"score":43.2,"score_display":"43.2","source_stated_rank":null},"run_fingerprint":"2605369b70f9323d0ce43d5ab7823a5b1c278b4b99eab65a19ceb25a78532f61","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Table 8.1, p.115; Figure 8.9.1.A, p.122","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4c1dd3ba4a09a30313d8edfd","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"anthropic-sonnet-5-hle","source_label":"reported","thinking_strategy":"adaptive"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"evaluation_at","evaluated_at":"2026-06-30","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"48ac4758f48c4cb99f37bff57e37ff058ac0d68ae99b5fd018f0dee34fd0c966","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/anthropic-claude-sonnet-5","source_model":"Claude Sonnet 5"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"de92dd50eda762a9af5a789c7edeb4db6ec6f051eb3cace840315241a93e24ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.2,"score_display":"43.2","source_stated_rank":9},"run_fingerprint":"a88e0c26fc821d5e6925da4c343fe43de6fcc7596df8b90ba85e00a7a1a3f0a2","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fbe0fdc94384bc26c7b61c70","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"f2948e2bf76eb4a26c876abfc5269c40877f81284a641c62a8cc5dae1302b28b","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","provider_default_effort":"high","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"7c3f637872fb46244f5b5da426d161cf1d53e48b97d141c1141f672acb42d4a9","result":{"confidence_high":59.325607,"confidence_low":55.451685,"sample_count":2500,"score":57.4,"score_display":"57.4","source_stated_rank":null},"run_fingerprint":"eff7c582b267b074e89bed66a292553461ef7af711ab16750815667fbee3d135","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_83831be53778e89901c24ca1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"db0d8264607e221b2b8e0191785aed7e8d3348d6dd9818de56fe927195f6f71c","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","provider_default_effort":"high","question_count":2500,"reasoning_effort":"xhigh","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"688227a378aa5a9d9a7aa9421a3075a0130edf6eaf926c4a61fd8491c2e8e6c0","result":{"confidence_high":56.543145,"confidence_low":52.64274,"sample_count":2500,"score":54.6,"score_display":"54.6","source_stated_rank":null},"run_fingerprint":"d1946809a4fd9fdb7b9e0aa256fc46146369d3eb39a5d5c920b030f8a6fe64a8","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_a3e2fbed1b6820c5de8165d7","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh","thinking_strategy":"adaptive"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"6d01ac47514d0ec024be49c36ac591c1567ddf3870ee7336530c410f9606c3d8","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","provider_default_effort":"high","question_count":2500,"reasoning_effort":"high","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"aa806c655efba8a3459511fb2c31f04999569fd04d0880d3a6f1a4f22f37520c","result":{"confidence_high":54.751131,"confidence_low":50.840277,"sample_count":2500,"score":52.8,"score_display":"52.8","source_stated_rank":null},"run_fingerprint":"414c832baa605f6d073310134241c1aac0524ed16c0b2e689a339f5e79e01324","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_d32b2c42170af1138c9b85b0","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"ac7ce45387197de7fa9ca056970e6e0d2b833840145ff0a786a7a8cfc5bd112d","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","provider_default_effort":"high","question_count":2500,"reasoning_effort":"medium","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"e6ac5a2f2a68f1723a6925f4b9c68204fc7c931eafd2b4094cadc3e54296bd51","result":{"confidence_high":49.159723,"confidence_low":45.248869,"sample_count":2500,"score":47.2,"score_display":"47.2","source_stated_rank":null},"run_fingerprint":"023d67226a1b09ddc2c223d83caade1660957aea5a2736283d89c216ecda07b9","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d0e15951c7e68535705c31f8","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"9a20300ff24f423aa24e954decb318341e20af26420c2691334e6b6ca8b6f37e","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","provider_default_effort":"high","question_count":2500,"reasoning_effort":"low","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"a1828867c18970142823c0f667c112f937bc0b7c190df942b85741fac6dfc1d8","result":{"confidence_high":38.406584,"confidence_low":34.634841,"sample_count":2500,"score":36.5,"score_display":"36.5","source_stated_rank":null},"run_fingerprint":"688be6f5a372fa246ab29ec8f7db31228304453850cfd89830954fd91e12ab5f","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c40bbe7a9ea062c9f214ec79","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e1d6d5914be1deec8a3fd954e31df2e27ae2d7ad5473a02d0c58850c9fc7d1ce","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · hle"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d11b49e4e05aeabf126d705998497fd9f0de58914840fc8ef821a3ad6cc23b9d","result":{"confidence_high":43.379255,"confidence_low":39.228167,"sample_count":2158,"score":41.28823,"score_display":"41.3","source_stated_rank":null},"run_fingerprint":"18cd5c41199b06e0b246f42608f3332251f8be263c9be870c0d540ece109085f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"770235b1430a72d639c8c9842088a049a3bb7179a9307b2a2e8cc0a4f5d549cb","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · hle"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-xhigh","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9a879dd1947a145110df4c5e7ff74ef21a2eb4ad4c77dc5acf306109340c7982","result":{"confidence_high":41.093471,"confidence_low":36.980778,"sample_count":2158,"score":39.017609,"score_display":"39.0","source_stated_rank":null},"run_fingerprint":"e9cf34f6d4404cf8719b732df59b2607e9107f0e039c0afd2a3a69db913ee7ec","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"729cac9d3ac5e4e4cca4c10d6813cdcb9b0f199d59c45361fdb94426d0c9e2e5","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · hle"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-high","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, High Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8a7be1c9056659993a926ff36d7ac5c2bb9c86fad63a3bf357b6f0fb1a00d757","result":{"confidence_high":37.773078,"confidence_low":33.732697,"sample_count":2158,"score":35.727525,"score_display":"35.7","source_stated_rank":null},"run_fingerprint":"1b3a474858b7e6808249791e3d03b8d68afed55cb9a9091b7bf5a359281a9e06","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e51cd7f59d8c43b8ee1e5e4e19dbf27f39d1eba9260494b5630844531a75037c","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · hle"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-medium","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Medium Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"be017269e0f81b28a3e53cbac7f5d3fcccd2078bd919e3d30bdf40c543b9434e","result":{"confidence_high":31.948786,"confidence_low":28.085288,"sample_count":2158,"score":29.981464,"score_display":"30.0","source_stated_rank":null},"run_fingerprint":"57b290a91c8f4e8ae5e293604b0d7885decca1aba18d9d2c5c0e791f53449763","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_017911fce805da3d633adae6","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"64821b06c7144ede3c776ac78e40ea1afd79502dd176f51bc573f147c568c972","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · hle"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-low","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Low Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7a39b58bbd0716d7bf29b9417155f675d0e5cf471cd15f660517bd5aba9b69ac","result":{"confidence_high":23.712964,"confidence_low":20.223724,"sample_count":2158,"score":21.918443,"score_display":"21.9","source_stated_rank":null},"run_fingerprint":"4c9c5bf3a0a59370d3315f3ef57799c7f8aa2af244969d53761591ed0d5aea79","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e672ac0437f08612f7384fba","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5136cc098c384ea299ffaff9f936bc9bb625993dc71072d723e27b3928a04f71","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · hle"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-non-reasoning","upstream_model_label":"Claude Sonnet 5 (Non-reasoning, High Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c733e848581817dbd29407b3b51d5186759928e80d22151f26222fbb9af85fe8","result":{"confidence_high":20.756569,"confidence_low":17.444268,"sample_count":2158,"score":19.045412,"score_display":"19.0","source_stated_rank":null},"run_fingerprint":"301714ffc5d793311b37c8210372924796b7fa85fb148dbbda02fc6fb1ed3b61","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6bca2a2ee2ca5f4a6d2be994","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:27:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:27:42.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"676c040c04170d7b889f56c472b3016c3d8311277394a465edf504917c007a8f","metric_details":{"best_score_across_scorers":"0.337","model_release_date":"2026-06-30","stderr":1.495508791865346},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LPo74T9L3CawnKX5LLUrSZ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLPo74T9L3CawnKX5LLUrSZ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/LPo74T9L3CawnKX5LLUrSZ.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"5861f9a63a01aed2d1972491628359bb645b56f3336acfb048a5a99449787c3d","result":{"confidence_high":36.63119723205608,"confidence_low":30.768802767943924,"sample_count":1000,"score":33.7,"score_display":"33.7","source_stated_rank":56},"run_fingerprint":"a906ed95b6001594ff97008396f113fda8d3ec9bc6726fcdb2c6c27e4ecb5a78","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:28:11.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:28:11.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"534bf933165b1c2763bfb08ca1622e0c30d923678f14e8551c1c78092d9d4239","metric_details":{"best_score_across_scorers":"0.329","model_release_date":"2026-06-30","stderr":1.4865395385928355},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Qw4tGw38pk4sdsEehhrLSR","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQw4tGw38pk4sdsEehhrLSR.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Qw4tGw38pk4sdsEehhrLSR.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"19a1f8b2dfa76b980508786ddb58eda6ef9dd82c31f6cb20e4e14b19d5ce9600","result":{"confidence_high":35.81361749564196,"confidence_low":29.98638250435804,"sample_count":1000,"score":32.9,"score_display":"32.9","source_stated_rank":59},"run_fingerprint":"6f3786c294f32042df457fb209efcb9699a6d411bde2c1046dc279edb565bb1f","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":95,"at":"2026-07-01T18:48:16.083261Z","basis":"evaluation_at","evaluated_at":"2026-07-01T18:48:16.083261Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"558b4b1207ddfd75f341251d872681cafab8d4781dc4463fb7d9739d4f8444a1","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-sonnet-5-default","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"23ed0e6170662177c11f1bbec13a83160e30a1994b2ef1fe047ed4bf69c9f559","result":{"confidence_high":35.33961471222776,"confidence_low":28.794319608441914,"sample_count":null,"score":32.06696716033483,"score_display":"32.1","source_stated_rank":22},"run_fingerprint":"af3cca369918eecda7d99b616b59bd2adf4ddd9a6d9a05fdb0c287f680bcd54b","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8166a5b71d36f7b192ec1df7","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d4d356d4db8fc6e79f5ce5c2f6d627785c04bc7475968ca914eefb4d70039550","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.077709,"date_is_proxy":true,"latency_seconds":63.234,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.224},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","few_shot_accuracy":87.374,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5","zero_shot_accuracy":90.404},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"f444c2af41e122cde27d7cc9dc44ee82c2ef4b7560c5878234ecd98539f587fc","result":{"confidence_high":93.24804,"confidence_low":84.52995999999999,"sample_count":396,"score":88.889,"score_display":"88.89","source_stated_rank":32},"run_fingerprint":"c1bcf6cdca9f0d5d519b1b321bc97b18cec541845de750468615a96bb63d7ab2","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ca20ff6bc26b307e97a29c007ef4275670eac03103d026b3ecf958d8032cc874","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · gpqa"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fcec8d16c0c45080d44b14ec1fa2e4d5f18ea3ee7a3837ffedf3e964f1a04385","result":{"confidence_high":94.331954,"confidence_low":86.325353,"sample_count":198,"score":91.111111,"score_display":"91.1","source_stated_rank":null},"run_fingerprint":"5661178e51436dbd3586681de5a27f15fe5f706851b70d0b3d03fb5960dd8f1e","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e02d34d9b782a1f2a90f60c28e3e4480ea18812e26d8374a49a1b6ca78f67b9d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · gpqa"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-non-reasoning","upstream_model_label":"Claude Sonnet 5 (Non-reasoning, High Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ea09a4d38eac6db2638a59ac78d2d9c498e6000eb1dbb44ff8879986dc6c0ced","result":{"confidence_high":84.976847,"confidence_low":73.881188,"sample_count":198,"score":80.0,"score_display":"80.0","source_stated_rank":null},"run_fingerprint":"c6c198498940e091eb3f14ed0d8425e6b3891c900db42e823e7943e0afcee033","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6bca2a2ee2ca5f4a6d2be994","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:21:47.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:47.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0ce91af07a0f9a9230985f76f986b6463bb99252eb2fb871afe7e587722e1191","metric_details":{"best_score_across_scorers":"0.803030303030303","model_release_date":"2026-06-30","stderr":2.8335609732463336},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FNXAt76ZUS3sWhQDFMga7U","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFNXAt76ZUS3sWhQDFMga7U.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/FNXAt76ZUS3sWhQDFMga7U.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"67f736e56a6979d2705761740f2e6f3b72601e0285f5bb2fd3c44b2b022280fc","result":{"confidence_high":85.8568098105931,"confidence_low":74.74925079546749,"sample_count":null,"score":80.3030303030303,"score_display":"80.3","source_stated_rank":127},"run_fingerprint":"a86c998dbc4db9f3c4c589a2fede90ef8fc5fbb63ce14b036fb9df5074dd4efd","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":96,"at":"2026-07-01T01:22:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-01T01:22:14.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"206f4e96d606e663a3e852351d329a93e589c4ee2c872ab65fcf1a369e8e24e5","metric_details":{"best_score_across_scorers":"0.9053030303030303","model_release_date":"2026-06-30","stderr":1.7838321313410213},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"HhiYxEx3Bf8YDPUh34K4DC","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d401c8aa227beacc10747f02da8398916c97e05b9af8cd56235d658b3e45fdfe","result":{"confidence_high":94.02661400773144,"confidence_low":87.03399205287462,"sample_count":null,"score":90.53030303030303,"score_display":"90.5","source_stated_rank":41},"run_fingerprint":"68b810d36d296a95e6ae95cdc6415baeff0dab04fe5819cfcd08a2d36ae39c30","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":81,"at":"2026-07-15T23:21:45.39117Z","basis":"evaluation_at","evaluated_at":"2026-07-15T23:21:45.39117Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a7b2488c5f16ef341bd63c7f6b4fc0be4d36d38ab1ec0c4c36406cf4c077a21a","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-sonnet-5-default","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ee50cffe44ccf711e37db497e6ef8c08d7cf2388c9f9a2e668d9c61d78a31685","result":{"confidence_high":85.39051652389983,"confidence_low":74.20544307205978,"sample_count":null,"score":79.7979797979798,"score_display":"79.8","source_stated_rank":20},"run_fingerprint":"0652d82c921f001288a8bae1af498c81f3f3496878994fce0f229074bcb23318","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7ac32f04e5800400ca7227b5","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a493d364c95cd50448157f2f5c11c234e1828ee7dc894c8b75e7d1f3313b133f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.042996,"date_is_proxy":true,"latency_seconds":25.191,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.368},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0084940a69e8fdb26871094b5a5e6ebb916247d3e4ee26cd0cb7b9e380aad6c2","result":{"confidence_high":88.26628,"confidence_low":86.82372000000001,"sample_count":null,"score":87.545,"score_display":"87.5","source_stated_rank":26},"run_fingerprint":"90b4c889bc6af375ae1e831818002be12cb4910b14ab13af6e6b3866925d2f51","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:21:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:48.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7ced96d1e8099670cdcc5b329ed1d506da13263b8477e62b7cfdf7dc02600571","metric_details":{"best_score_across_scorers":"0.8","model_release_date":"2026-06-30","stderr":6.030226891555273},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"XRutLYUVUePe69APTSTw8S","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXRutLYUVUePe69APTSTw8S.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/XRutLYUVUePe69APTSTw8S.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9a0868ac73fde87850c5807de8f84ccbf2a2cc8c57e6b1ff438531fc033606e1","result":{"confidence_high":91.81924470744833,"confidence_low":68.18075529255167,"sample_count":45,"score":80.0,"score_display":"80.0","source_stated_rank":110},"run_fingerprint":"10cd811349e2b0b6036fb3f30897449cc23a43160f03fd5b2b34e8cfb5dab0de","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":95,"at":"2026-07-01T16:06:23.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-01T16:06:23.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"d4ecfe6ec9ce89c376fc348e34dfba2ce849bd76760c853e5817c4034a854e11","metric_details":{"best_score_across_scorers":"0.9472222222222222","model_release_date":"2026-06-30","stderr":2.2644349282618412},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"T2wFfuSkUdyjpyqurxNmNK","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"abe7efaa3ff80ee2b1b3165edf60af41c1ca4ddecc2cba34e72dae473a464236","result":{"confidence_high":99.16051468161542,"confidence_low":90.283929762829,"sample_count":45,"score":94.72222222222221,"score_display":"94.7","source_stated_rank":47},"run_fingerprint":"97187d43ac6288d89788eb12e179be68b4877cdbe3396f0ee52368a9abcdf675","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-06-30T21:54:08.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-30T21:54:08.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"065b730377e1c75513816a1345a4be8180c5c4f5f42645714a18f2361ac63d17","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.656140350877193","leaderboard_private_problem_count":285,"model_release_date":"2026-06-30","public_example_count":10,"stderr":2.818576398907258},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"V6YuGpcu8MCbJNXKoJMA6b","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FV6YuGpcu8MCbJNXKoJMA6b.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/V6YuGpcu8MCbJNXKoJMA6b.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Sonnet 5 (max)","thinking":null,"upstream_model_id":"claude-sonnet-5_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"6f9eaaa7835e7dbe8796b8a67e624a1fa6c34db8f6fda1c04fc3d72e22630fae","result":{"confidence_high":71.13844482957752,"confidence_low":60.08962534586107,"sample_count":285,"score":65.6140350877193,"score_display":"65.6","source_stated_rank":30},"run_fingerprint":"0db227b85b934f375ef1b11e368ce754312bf874ba1a41edb0d4102210d8be03","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-06-30T21:54:08.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-30T21:54:08.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"236069a02b42086b181c502d31c557d30e0426b5b95e5d4769ecc77803084438","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.2926829268292683","leaderboard_private_problem_count":41,"model_release_date":"2026-06-30","public_example_count":2,"stderr":7.194088392074452},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"bqiVinHXFcgyFGPstJwXJN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbqiVinHXFcgyFGPstJwXJN.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bqiVinHXFcgyFGPstJwXJN.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Sonnet 5 (max)","thinking":null,"upstream_model_id":"claude-sonnet-5_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"dbff2f02936d4e952441a601d60d08dd5c4312aaa3c35b3fd6c65947d0f45232","result":{"confidence_high":43.36870593139275,"confidence_low":15.167879434460902,"sample_count":41,"score":29.268292682926827,"score_display":"29.3","source_stated_rank":34},"run_fingerprint":"9baf803e9753e44e395de46ac46e68560f7c15bc6a0609a4bd5e66b4e9bf7534","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"02f251927b199f0367714cc54e0074828090822536f98c428164fe9ddbafefff","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.033133,"date_is_proxy":true,"latency_seconds":18.817,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.903},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8d10c431fb39ea981a29f2c36dc7ce479d08e819e4b9f018ce2d7a5f41ab06da","result":{"confidence_high":84.77588,"confidence_low":81.23612,"sample_count":null,"score":83.006,"score_display":"83.0","source_stated_rank":35},"run_fingerprint":"82b01c6bde17e68c1189a3dab7a319dba1245c316c64dabf08064facd7e87de5","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2751ee7e1061f2d99648ddaa9f5f3dd5d25253c41bd9bdcbf9852f65f4cb663b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · mmmuPro"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"da45c7fe4cf9f181d93e9d4c75d4362c9c1917c0f5d7cef61948a5a663966290","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.283237,"score_display":"77.3","source_stated_rank":null},"run_fingerprint":"80024e8a0ccfc7e401c7a349789641615c81bdbaa0b9eaec3f7efcc0d36056d7","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0038d321b46d9aec4ef2bdf040e5154c3ec62bb718154ab69df0699444b29ced","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · mmmuPro"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-non-reasoning","upstream_model_label":"Claude Sonnet 5 (Non-reasoning, High Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d63f7f946dc59118438e826ee8afeb62402511ff61d51e5a0135d0cd6f5b66a5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":71.907514,"score_display":"71.9","source_stated_rank":null},"run_fingerprint":"f5a50d29b5c670c6df2da3b1b66f9ba84dc1fcce828a05f022b413de777e6c6b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6bca2a2ee2ca5f4a6d2be994","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b6139ead1653d5373ecb435ff27302f60a4bb96cba1cd9995be58edb10872807","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-sonnet-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c8a8bb2f7d7f4090d5bef01b45161f888bcf65d20d5e142e5a9fa7f6f8e5e422","result":{"confidence_high":27.209102,"confidence_low":9.907613,"sample_count":71,"score":16.8571428571429,"score_display":"16.9","source_stated_rank":70},"run_fingerprint":"79d2b19c6d361682276cc973f972931a30f85803f6ffe9c92a9c5ab4e36f02c8","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5493fb1e39d1edd4a05b76fb1e7bd9a37fcfa40444ba06d6d21bb2bda90c5cc4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5 (Non-reasoning, High Effort)","source_effort":null,"source_model_version":"claude-sonnet-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5a09465b86d4d355496ccaaf55aef098a778ed2b702ac7ffab949ba0d3f46797","result":{"confidence_high":7.127519,"confidence_low":0.173844,"sample_count":71,"score":1.1428571428571401,"score_display":"1.1","source_stated_rank":140},"run_fingerprint":"e06635bcd8c2b580bdb913ee62b7bdb58067700d68a8575c9c3f00fe13af48e1","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e48ccb949a4368bdc940ea62da1e63b897ceb6ecaceb64a1b4b0191917a5d541","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · critpt"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"06fc86a4235191aa5bf966a3e588ae930ec7d8efd2c0ccca61f4753bf56790e4","result":{"confidence_high":27.2928,"confidence_low":9.869992,"sample_count":70,"score":16.857143,"score_display":"16.9","source_stated_rank":null},"run_fingerprint":"521501294559c7e5b25f449c91b9ad6848896b027c4a2694c847848b541ee94b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":23,"at":"2026-09-11T17:49:02Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-11T17:49:02Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"136650d530896f83d15e90f20a9016a2715a65e6f51507623b9f6870cd8715a7","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · critpt"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-xhigh","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b37578326323ac660c9bba807280e0ca4a66f4a7394920038ad08d4fec7e9da9","result":{"confidence_high":25.660281,"confidence_low":8.79401,"sample_count":70,"score":15.428571,"score_display":"15.4","source_stated_rank":null},"run_fingerprint":"31ae2735194f0a1ff4a5673cf730e65bc860ea13683e305d4eeb01e6078561f7","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":24,"at":"2026-09-10T20:32:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-10T20:32:06Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"80ba7e7668d2f49527d9418a1fd9c75c2817b2c963ba945687d889dd530a8a23","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · critpt"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-high","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, High Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"e18fb28019646574e3c2d6c29c4cd866fcd3cab5d9324749afb46ea5f599e5f7","result":{"confidence_high":25.331226,"confidence_low":8.581365,"sample_count":70,"score":15.142857,"score_display":"15.1","source_stated_rank":null},"run_fingerprint":"1e5b921b290aa0ab2629da5d6be9336c6680258d4069e28731964edcf12411ef","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eedde0296df29a1d307bc39a85908e3b5e7cff31b75003f38b7edd2b2b6bdd3f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · critpt"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-medium","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Medium Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"eb80da12e0cb9e19e6dc2da22676905763247fec3249db4b1ea7557b01ba353c","result":{"confidence_high":17.465878,"confidence_low":3.987613,"sample_count":70,"score":8.571429,"score_display":"8.6","source_stated_rank":null},"run_fingerprint":"1158a705730d7d4649f93adbd0598a7dfb109c02924943a9ece8d47384f62166","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_017911fce805da3d633adae6","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":28,"at":"2026-09-07T08:30:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-07T08:30:17Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f1cb5ad1118b28b662bb76acd72a5405d00f1e406ac2591ffcd628b32696898e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · critpt"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-low","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Low Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5fd82f8939b5f4dabef56b75cfc29ece6dc61266a85d7051483e9e2c768fb0c7","result":{"confidence_high":12.252863,"confidence_low":1.616826,"sample_count":70,"score":4.571429,"score_display":"4.6","source_stated_rank":null},"run_fingerprint":"a92213d9bd009345003bb2f0ea0b939454ad6f0a874d595d035092ff5cf8e10a","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e672ac0437f08612f7384fba","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e2269a5afabb5984afc93c3904b101a7d21c736b13bc9b9c17ba5c0b11ba6b99","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · critpt"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-non-reasoning","upstream_model_label":"Claude Sonnet 5 (Non-reasoning, High Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"9a14ed5752b33b5daccb3698688d8c57d124be88b62edb68bb3ba3f99ef6875c","result":{"confidence_high":7.197254,"confidence_low":0.172034,"sample_count":70,"score":1.142857,"score_display":"1.1","source_stated_rank":null},"run_fingerprint":"85e0026b0a0e51923d426bcf2df88f180599ba63f6cb5138743a71ee669acf8e","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6bca2a2ee2ca5f4a6d2be994","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d325caa55bcd8849055abd9719e28d3d22c76232f634b37b7d2d392b2725a4f8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-5_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"cd468ee7668bb194c17795fc612b46112ede3da7738b362f51a2954fff51d4e2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.6,"score_display":"60.6","source_stated_rank":30},"run_fingerprint":"10215e02130ed92d83fe426db825b890c9ccf79fd59e475a5ef101e5d4f9f8c1","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7249740ffb460d0140f7c11a","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-06-30T18:32:59.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-30T18:32:59.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"718858d1d2a4e0d015ba1b03f182eca69097d55e68c762ba05d700936c54f32e","metric_details":{"best_score_across_scorers":"0.16","model_release_date":"2026-06-30","stderr":3.6845294917747067},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"czNLsHc7tBqBWtWaoTSLXh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FczNLsHc7tBqBWtWaoTSLXh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/czNLsHc7tBqBWtWaoTSLXh.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b20217346229eab69b2086fbdbffed1a31fb95df8002b58af4a037765a840988","result":{"confidence_high":23.221677803878425,"confidence_low":8.778322196121575,"sample_count":100,"score":16.0,"score_display":"16.0","source_stated_rank":109},"run_fingerprint":"f9f274ffdf5a1009c4f8b428eeee38ebdea23e1c7faba5bd63947a4ef7f88a1a","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-06-30T20:26:11.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-30T20:26:11.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"33551b9b9e746efdef382b004dd88d7e1facc5e60273092d38cedcc873d2b284","metric_details":{"best_score_across_scorers":"0.35","model_release_date":"2026-06-30","stderr":4.793724854411022},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"DAyGUk5kLRdwNkSPZgqmos","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"fbbe8317c850dd3d54105d6bcec6d4957e1ca662bf8519bfb19d3a927c05eb19","result":{"confidence_high":44.3957007146456,"confidence_low":25.604299285354397,"sample_count":100,"score":35.0,"score_display":"35.0","source_stated_rank":40},"run_fingerprint":"2935d8617eec1812df92cd1ef010e6c145026b4aa29bb1a8e7ca9178f28c62c7","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a9cbcf7fc3d8a5a1b8f3ff79dede00ceb3655d209eeccedbc98dea86a23e04c2","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.489899,"date_is_proxy":true,"latency_seconds":962.366,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.804},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"b7de84535c9f4a919049691676aaa2e9e6c0cceba281f5ad581d351452ac8303","result":{"confidence_high":83.13583999999999,"confidence_low":76.06416,"sample_count":500,"score":79.6,"score_display":"79.6","source_stated_rank":25},"run_fingerprint":"485b31b1a3e31f014e98007c70c759f5a3931b606703fee7dc399e84013d6de1","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ed49ef6cb40ad079db5a0835","split_or_window":"verified-500","unit":"percent","version":"Anthropic 500-task evaluation"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"ed2c6e9a49e5bf0713ea90d00f46f20516e794943238d12da1fbf4c80d35f793","metric_details":{"comparison_warning":"Vendor run over 500 tasks; do not equate it with Epoch runs over 484 tasks.","comparison_warning_code":"swe-vendor-500-vs-epoch-484","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.2, p.116"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"swe-vendor-500-vs-epoch-484","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; five-trial average","question_count":500,"reasoning_effort":"max","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"not reported"},"run_count":5,"scaffold":"unspecified agentic harness","tools_allowed":"not reported"},"protocol_fingerprint":"d5cc9a90256ee91e03c40fe9632ce24357887404d2ae52765ce6a78ae3e40b84","result":{"confidence_high":88.043907,"confidence_low":81.81932,"sample_count":500,"score":85.2,"score_display":"85.2","source_stated_rank":null},"run_fingerprint":"e0dfa8c8ce25b948ab6d862f76dd7c609689de30c1eee3dacfec90e0f1047af2","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.2, p.116","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_83831be53778e89901c24ca1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1fb2caa7d4decc0d4cc7ea4ffbf25902cd01e205d85eeb456548710b3c69fe2b","metric_details":{"model_release_date":"2026-06-30","pass_at_5":74.77477477477478,"potential_contamination":true,"sem":0.9448728361893257},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"contamination-risk","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"Sonnet 5 [high]__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"94196f1040cab460930695009cb99a0ad38c89a5898c53c3815a04f1b2eea6fa","result":{"confidence_high":58.60870751568784,"confidence_low":54.904805997825676,"sample_count":null,"score":56.75675675675676,"score_display":"56.76","source_stated_rank":6},"run_fingerprint":"64cef58baa68cbe36a9f59574a3c5acd45ef7c5539dd8f08c1e8cf2ea31dd092","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"contamination-risk","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"e110c10f9a8ac05f173262f4589ef8f3131f54d3ea01f9fabc58e424bbeb3400","metric_details":{"ci_attempted":442,"ci_half_points":4.2369164701743856,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":238,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":268.4547511312217,"mean_cache_tokens":71991474.90950227,"mean_cost_usd":26.399858950791852,"mean_duration_seconds":4803.543157789592,"mean_input_tokens":72415756.48642534,"mean_output_tokens":214117.54977375566,"median_agent_steps":260.0,"median_cost_usd":23.277376725000003,"median_duration_seconds":4345.498858000001,"median_input_tokens":62591138.0,"median_output_tokens":203917.5,"median_output_tokens_to_pass":203255.5,"median_peak_context_tokens":409978.5,"n_attempted":442,"n_passed":238,"n_tasks_attempted":113,"n_tasks_passed_any":89,"pass_at_4_points":78.76106194690266,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_sonnet_5_max","source_model_version":"claude-sonnet-5","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"c661237bdafd6d2237be5b74378ff13ee23b9c201c5eec402ab9f08e20a77824","result":{"confidence_high":58.08307031632823,"confidence_low":49.60923737597946,"sample_count":442,"score":53.84615384615385,"score_display":"53.8","source_stated_rank":42},"run_fingerprint":"84c961b29c5c2ac49ea59f3160fe1a15043f1761b204d4a151e8b6fb1a7da2c3","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"ccd9a488b58618dcff65f7b4e39ecbe04bd62baca1510a0f8b4a764688fb719e","metric_details":{"ci_attempted":451,"ci_half_points":3.4545489916778647,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":224,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":185.53215077605321,"mean_cache_tokens":30442947.99556541,"mean_cost_usd":11.890634540022173,"mean_duration_seconds":2414.4529098159646,"mean_input_tokens":30695627.87804878,"mean_output_tokens":120698.59201773835,"median_agent_steps":174.0,"median_cost_usd":9.965361750000001,"median_duration_seconds":2229.375982,"median_input_tokens":24771332.0,"median_output_tokens":113150.0,"median_output_tokens_to_pass":112015.5,"median_peak_context_tokens":240780.0,"n_attempted":451,"n_passed":224,"n_tasks_attempted":113,"n_tasks_passed_any":85,"pass_at_4_points":75.22123893805309,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_sonnet_5_xhigh","source_model_version":"claude-sonnet-5","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"31eafca99777ec3fd901d38e5309b02f1a177a8e4253c84c76ab710a2be3c947","result":{"confidence_high":53.121954756644605,"confidence_low":46.21285677328888,"sample_count":451,"score":49.667405764966745,"score_display":"49.7","source_stated_rank":50},"run_fingerprint":"32d91caa9b872c21d67e00fd346037106b7d1126bd8ebde78e16c7206fd7643b","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"c59ce00d441d1667fdf13a528175a5ed91583bb6dcb34af3840e908d52c2cd4e","metric_details":{"ci_attempted":452,"ci_half_points":4.50631487021484,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":218,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":146.57649667405764,"mean_cache_tokens":18068889.902439024,"mean_cost_usd":7.425560161197339,"mean_duration_seconds":1728.2193087920355,"mean_input_tokens":18254491.34368071,"mean_output_tokens":87294.75609756098,"median_agent_steps":138.0,"median_cost_usd":5.833649100000001,"median_duration_seconds":1539.1866639999998,"median_input_tokens":13842595.0,"median_output_tokens":77993.0,"median_output_tokens_to_pass":76281.0,"median_peak_context_tokens":174563.0,"n_attempted":452,"n_passed":218,"n_tasks_attempted":113,"n_tasks_passed_any":90,"pass_at_4_points":79.64601769911505,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_sonnet_5_high","source_model_version":"claude-sonnet-5","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"210ad6db67749d38f1b1bc54ef3a06e179b89dfbc75832e2113533721290de3d","result":{"confidence_high":52.73640336579006,"confidence_low":43.72377362536038,"sample_count":452,"score":48.23008849557522,"score_display":"48.2","source_stated_rank":52},"run_fingerprint":"4691a7201c4d027380374896f216bd80e5ec0044e1d4e770aa73544333af54d4","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"eb9f0dcf1e0d77c1e28a6b2c052c6b3533d2edb0f11632e835bdb1282ea78498","metric_details":{"ci_attempted":450,"ci_half_points":3.127576206329727,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":179,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":107.61111111111111,"mean_cache_tokens":9158953.993333334,"mean_cost_usd":4.079036229,"mean_duration_seconds":1121.270593551111,"mean_input_tokens":9286962.313333333,"mean_output_tokens":56816.753333333334,"median_agent_steps":100.5,"median_cost_usd":3.3573110999999995,"median_duration_seconds":980.8059874999999,"median_input_tokens":7084139.5,"median_output_tokens":50414.5,"median_output_tokens_to_pass":49405.0,"median_peak_context_tokens":118790.0,"n_attempted":450,"n_passed":179,"n_tasks_attempted":113,"n_tasks_passed_any":73,"pass_at_4_points":64.60176991150442,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_sonnet_5_medium","source_model_version":"claude-sonnet-5","source_reasoning_effort":"medium","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"f2c8ce1f988dd8e1d8783b9785bc7c3613d71383aaf72e5fe4994344279cfd4d","result":{"confidence_high":42.90535398410751,"confidence_low":36.65020157144805,"sample_count":450,"score":39.77777777777778,"score_display":"39.8","source_stated_rank":59},"run_fingerprint":"7e3fcc6bb4ec29a42632b603be9cbeaa3599c2e58c63ad0c2fb28dcd0f7383f3","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_017911fce805da3d633adae6","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"ef7f92729cb7429396ee099ebfeaf717f0463a9982b69a94225bf345d82ff6e5","metric_details":{"ci_attempted":449,"ci_half_points":1.1260175414427496,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":137,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":76.88864142538975,"mean_cache_tokens":4449592.527839644,"mean_cost_usd":2.1865550832962137,"mean_duration_seconds":734.9105084298441,"mean_input_tokens":4534475.699331849,"mean_output_tokens":35595.08240534521,"median_agent_steps":70.0,"median_cost_usd":1.6764955500000003,"median_duration_seconds":624.639023,"median_input_tokens":3126877.0,"median_output_tokens":31478.0,"median_output_tokens_to_pass":32131.0,"median_peak_context_tokens":76944.0,"n_attempted":449,"n_passed":137,"n_tasks_attempted":113,"n_tasks_passed_any":65,"pass_at_4_points":57.52212389380531,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_sonnet_5_low","source_model_version":"claude-sonnet-5","source_reasoning_effort":"low","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"72475f7f86412cf896259c21b1e3c3f337580e4d7735309d687ce06cb7e0fe00","result":{"confidence_high":31.638266984649878,"confidence_low":29.386231901764383,"sample_count":449,"score":30.51224944320713,"score_display":"30.5","source_stated_rank":64},"run_fingerprint":"0b698aaf6389977d66c2e8c601d0c9f852920137462ed36e3acf3ef1ba416b8b","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e672ac0437f08612f7384fba","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5c49853ae0ba561d86e4138471b3a09e4830e4fadbd9c21c628ffda491cc871c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"268.4547511312217","mean_cost_usd":"26.399858950791852","mean_output_tokens":"214117.54977375566","model_release_date":"2026-06-30","notes":null,"pass_4":"0.7876106194690266","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-sonnet-5 (max)","source_effort":"max","source_model_version":"claude-sonnet-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"5790d85c3657540c0ed41535f9f273e4d8372e1f01229538fc8570bf67207288","result":{"confidence_high":58.08307031632823,"confidence_low":49.60923737597946,"sample_count":113,"score":53.84615384615385,"score_display":"53.8","source_stated_rank":41},"run_fingerprint":"b7b41dcfcb15ebbc1f569a64f3c4e365753bb8eba4074e1578b8c4f73d81a20f","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6ab0d5ccd693ba49eba76eff88ebbd59737d1478813d1387d0f35c55e8812eec","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"185.53215077605321","mean_cost_usd":"11.890634540022173","mean_output_tokens":"120698.59201773835","model_release_date":"2026-06-30","notes":null,"pass_4":"0.7522123893805309","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-sonnet-5 (xhigh)","source_effort":"xhigh","source_model_version":"claude-sonnet-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"7d798d4a8704fe906ebb629830af3e1bbf6b0e9e29b4fe711898bb3dcab3944e","result":{"confidence_high":53.12195475664461,"confidence_low":46.21285677328888,"sample_count":113,"score":49.667405764966745,"score_display":"49.7","source_stated_rank":48},"run_fingerprint":"f6fe4786c3ef4cafe59b4540420a599a3f4539c9c17f111df58affd57d03d350","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"870f54dbf186247de3d93bba84ce11a7c7f9cedb75422b365085c3a125f46c50","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"146.57649667405764","mean_cost_usd":"7.425560161197339","mean_output_tokens":"87294.75609756098","model_release_date":"2026-06-30","notes":null,"pass_4":"0.7964601769911505","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-sonnet-5 (high)","source_effort":"high","source_model_version":"claude-sonnet-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"26c1803de343ad621c50c41790373adb1ebe83c20e99ddf95f50bab644cdb6d6","result":{"confidence_high":52.73640336579005,"confidence_low":43.72377362536038,"sample_count":113,"score":48.23008849557522,"score_display":"48.2","source_stated_rank":50},"run_fingerprint":"1944b6c763186e5a4c798c76f63372d5852646c390f46a5bc222d7174f3a308f","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e4b4736fe132438762f1588dfa3bf54d7b4a9c841b78e02161fd75bf6b342ca3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"107.61111111111111","mean_cost_usd":"4.079036229","mean_output_tokens":"56816.753333333334","model_release_date":"2026-06-30","notes":null,"pass_4":"0.6460176991150443","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-sonnet-5 (medium)","source_effort":"medium","source_model_version":"claude-sonnet-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"98292a2ca1a83047a15f279c295165642eb062aa52a90d50c8627436ee938d7d","result":{"confidence_high":42.90535398410751,"confidence_low":36.65020157144805,"sample_count":113,"score":39.77777777777778,"score_display":"39.8","source_stated_rank":57},"run_fingerprint":"782148b8984abf4b59c7baffd04fd3a3c57cd40fc509315fe465bad1290c514d","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_017911fce805da3d633adae6","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fb85615b0e3f3f213d238c9c2cf1ceeb64963de5805dbe9edaf337a46612aa62","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"76.88864142538975","mean_cost_usd":"2.1865550832962137","mean_output_tokens":"35595.08240534521","model_release_date":"2026-06-30","notes":null,"pass_4":"0.5752212389380531","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-sonnet-5 (low)","source_effort":"low","source_model_version":"claude-sonnet-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"eec71199b20ffb7647168f7a4cf796cfa36ce16e2408733521fca32992e8a2bf","result":{"confidence_high":31.638266984649878,"confidence_low":29.38623190176438,"sample_count":113,"score":30.51224944320713,"score_display":"30.5","source_stated_rank":63},"run_fingerprint":"7a1d7071d54c275f416bce19809ff2ea84f57f9a32f952853e83faea64f3727c","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e672ac0437f08612f7384fba","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_6323abbd504101af2cdaac58","split_or_window":"vendor-public-own-harness","unit":"percent","version":"SWE-bench Pro · Anthropic provider run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"3f5360c1f2b87af3d96aadd78da4315add4944c62bb2e28e3621d6a9fc586c5f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.2, SWE-bench Pro capability evaluation"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; max effort; five-trial average; exact public-task revision not restated","reasoning_effort":"max","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"software-repository shell and code-editing tools"},"run_count":5,"scaffold":"Anthropic agentic harness","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"3793fd31f27dc20eba5e39f12b3454bb06e0f8a244cdf94b80657a4f09718617","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.2,"score_display":"63.2","source_stated_rank":null},"run_fingerprint":"f34af5c4e9e97eb6775f2330867aeb23c84da419e090b1949ce6035243dc8e21","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.2, SWE-bench Pro capability evaluation","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_83831be53778e89901c24ca1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8e89fc1380bb6ad39bef7cddceb18c10b6b9332b255cb7bf55d753e6f8a573c2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"7.17","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"steps_per_task":"140","tokens_per_task":"149257","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5 Max","source_effort":"Max","source_model_version":"claude-sonnet-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"c3eb65951e88a1203f8c118af827c0dfe2d54eb2e33c3178334c6e00a9ebc071","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.1,"score_display":"34.1","source_stated_rank":36},"run_fingerprint":"dc3e38f766800c3d8a2359b0c349280a4a1154f49100921aab474403078854c9","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f0b5a389482227c6fcdf4f535af04f36965f5e5a26e9aeb2e019172c8aec335f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.55","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"steps_per_task":"102","tokens_per_task":"83373","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5 Extra High","source_effort":"Extra High","source_model_version":"claude-sonnet-5_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"67e51440f5332b21785d4813401b82d3f080c0f3b1ca4fa38caa3ff8c311d19a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.0,"score_display":"32.0","source_stated_rank":43},"run_fingerprint":"71042df01ca2a3bec505192be7c414a458f09a6cd85abb1e1558b6c5c0bd1a1d","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a4864a61d058bfcdff90633f5d344b556cae65ea7c5cd5a8c29f0bc57557d535","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"3.48","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"steps_per_task":"85","tokens_per_task":"61146","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5 High","source_effort":"High","source_model_version":"claude-sonnet-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"713ab59ab15c09f1c702806ee090b43fae43b4ec55707052647dfec721774210","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.8,"score_display":"30.8","source_stated_rank":45},"run_fingerprint":"b63418ef61b3b4efa2f3342d2d3a79d92a9215ed335c98b5afd9540e1def4f0e","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"abf5ffef1b673724fd741f2706cfa768c908eb957e596bf3dc731d02024b820f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.31","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"steps_per_task":"65","tokens_per_task":"39114","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5 Medium","source_effort":"Medium","source_model_version":"claude-sonnet-5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"0b4da280337da94dbabbf7a20f63e6f6b5057160a2579e8445a0420c5f7082b3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":28.000000000000004,"score_display":"28.0","source_stated_rank":49},"run_fingerprint":"9286a94dec9e7a4a85e612b7d3599eada7011872abf0c94edee2b01670108fbe","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_017911fce805da3d633adae6","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cfb804739b226fa88ffabafc50c364ca943fd57340ba7821d1685d5222bda63b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.39","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"steps_per_task":"46","tokens_per_task":"23772","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5 Low","source_effort":"Low","source_model_version":"claude-sonnet-5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"fc686fda650a90a443514bce3f1979054487e518d35ed33ed6616f3661b67619","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.1,"score_display":"24.1","source_stated_rank":55},"run_fingerprint":"8cb5ecdc55c0d386411a25ae7e78ed8d8da2a1c91bae0dbf2c547d154bd4b9e7","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e672ac0437f08612f7384fba","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":85,"at":"2026-07-12","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-12","status":"fresh"},"measurement_id":"eedeb92df7314dfe81c16bfcd5fa57f88ad32abec5f5368e67ec4ae62aeca5e0","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-5_unknown","source_row_date":"2026-07-12","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"18b94ab8f540d424f8ed062411389259b95dbf9a9d86613682c99b284484d325","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.25,"score_display":"37.2","source_stated_rank":6},"run_fingerprint":"602f7dea96d4c8601a39d3f3cace6f92ddadd899138a248a5888a005edb01844","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cf91388c08be94a36c0d7705","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d140fcdb9e9069679c9c79190731ef9dd01f62a5c62044fb371f064793b730d6","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5","source_effort":"xhigh","source_model_version":"claude-sonnet-5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"f4bcc4c61f272444f58d4638e48d10753414346ab0260cfc3f2575c95d1ca708","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":42.730000000000004,"score_display":"42.7","source_stated_rank":16},"run_fingerprint":"f867e7dab62890d81041b2d54373b5816625425bc99e998bc41e18126c79f326","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a69be95a08f65f0301b13536102c00870a4d1fb1826cd75234fae5369e9ea9de","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":12.889951,"date_is_proxy":true,"latency_seconds":3583.884,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.754},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":null,"source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"985c31f53c9a762184d5ea0b5467fbf5f8f8628a5b0972ab3f62133a8d7993f1","result":{"confidence_high":50.39784,"confidence_low":39.60216,"sample_count":null,"score":45.0,"score_display":"45.0","source_stated_rank":28},"run_fingerprint":"f3a13b565be2f5d6a6480fa302ab8219bbe239d73032f664615456e8ef2fb1f0","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9d6acc70fc0024fa20d3f9a05ec584db195dfddfb2b3d6e197bc0c500cca99c6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.094655,"date_is_proxy":true,"latency_seconds":77.021,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.088},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"d2e8ef2488f1c4d064f17a50c5f229e440d6c6148de542f9817e11a77321298e","result":{"confidence_high":84.56148,"confidence_low":80.29652,"sample_count":null,"score":82.429,"score_display":"82.4","source_stated_rank":53},"run_fingerprint":"1b293cc029498a9173b5f6b3c0ae7fd5626f455082fd130e3ebef8be5cb7cfd7","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2a8c42d878c3ab00924d8cb906854b20706357a6d422eaf39c66c1d4463db1f5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-sonnet-5_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fcb876f2caeaf3498f1d2ff110b7eb80fdf6e8769742151b4e04f79acc0a5133","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.587962962963,"score_display":"53.6","source_stated_rank":59},"run_fingerprint":"89a32dd1da312b42cd1921de4fe5258111015d9d528551a49421ca01f39f428c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1c399030a871c1eaff99372cc9ea7d6f03e3d269210a403971004f56f99dfc6a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5 (Non-reasoning, High Effort)","source_effort":null,"source_model_version":"claude-sonnet-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"3c881e6d5e6a90e8b95b44fbe3ac3a8837e2a64734d29b4d014c811e51e89265","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.6111111111111,"score_display":"48.6","source_stated_rank":102},"run_fingerprint":"3e47eefffdbc5f5bef850fd68c32435deb3532efe1bace672dfbf5ec2518db5c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1f3b478cd2f45b33cedb2c80ef8a839271d23c44bab5c9459fcf8faadde361b5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · scicode"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"dc92a4bf5d4c85c066521e6f45b2763253f41ca611a027200760dfbca38e0363","result":{"confidence_high":59.941807,"confidence_low":48.510265,"sample_count":288,"score":54.282407,"score_display":"54.3","source_stated_rank":null},"run_fingerprint":"3c2567a6261a17076c4f3f37f075b3b84bb8b07376d04df8ba01288bf1f10d0d","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-07T14:33:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-07T14:33:47Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"232c2f184da71059a04aa6b5d05f6ae5fa3cb7463b0fcc10d3dee87644e4d14b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · scicode"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-xhigh","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1f4cbb39571215cb2e058871db68a50a306477c22e2598a102b2d27b82624713","result":{"confidence_high":59.715565,"confidence_low":48.27964,"sample_count":288,"score":54.050926,"score_display":"54.1","source_stated_rank":null},"run_fingerprint":"30a7dc5235c96f6ab6f06b241ced318665b72feb206e3205dc4d0b0ac26d3e73","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-07T14:33:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-07T14:33:47Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0e47275cacf739bbd924ecc4d74d2dd050eceb9a5bc6570110f46b57159900f2","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · scicode"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-high","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, High Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"23dda1e6fb8436a89dd2056ed51264d62f700258e282c84935d06fcbe8432eca","result":{"confidence_high":59.941807,"confidence_low":48.510265,"sample_count":288,"score":54.282407,"score_display":"54.3","source_stated_rank":null},"run_fingerprint":"80350a1fba41d0360d3442b964353a032a05dac6878c85c2f766f079178b53a1","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"490782fa51a9ce8f5650139e04a6dca5a60c89f6eeea3d0657ddb57c32ac7af0","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · scicode"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-medium","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Medium Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"4d4c6f8decf9a3407d2068a103775d79b799cc93cb826fe95096fb337226d4b8","result":{"confidence_high":57.332639,"confidence_low":45.865442,"sample_count":288,"score":51.62037,"score_display":"51.6","source_stated_rank":null},"run_fingerprint":"09d84841706bde1014cb1a3d11b535b1632cc74e3a06522007c789b4096ad506","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_017911fce805da3d633adae6","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"665016b9d6006528caff02a42886a5f9e0f0bbe0bd1215e4b14250f791cfd346","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · scicode"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-low","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Low Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"051b9bd85572903b7fb0be0ccd523f79a23a292ae1c555667c6ab0075567946f","result":{"confidence_high":55.850775,"confidence_low":44.37766,"sample_count":288,"score":50.115741,"score_display":"50.1","source_stated_rank":null},"run_fingerprint":"3eb44fab23f2a7e93d884525521d94f8e91c36bf869a2950eb65093b4d1e13ed","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e672ac0437f08612f7384fba","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bba28a12090dcad99f9d8217c34d0c636d800aeea98067276644803417afd490","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.96046","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-sonnet-5-high","source_effort":null,"source_model_version":"claude-sonnet-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"7ddf2d974e629787f018a6349f86d2a4ff76f2fb2671251babcf7840475de123","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1463.12,"score_display":"1463.12","source_stated_rank":20},"run_fingerprint":"398075636ee5ccf1880021137559c1012a9650f46018ccd65592d32b11e988a7","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"21ee814da175603538548892130e8a244df5d074b7c83afc9c419b0a938af2df","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":25.38542,"date_is_proxy":true,"latency_seconds":4001.424,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.046},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"cacb325e9d02103958b058aef79a8a020b266ba7916f7fe34c1c69306ba3f553","result":{"confidence_high":87.29516000000001,"confidence_low":75.35484,"sample_count":null,"score":81.325,"score_display":"81.3","source_stated_rank":20},"run_fingerprint":"3402fa1de5937a0bbcec758c2ee694bd8d69307096110d9ba7998fbc7db6ee86","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"15294d62220fa64755a9776a7c159d9209425855ede6aab76281bf892562e70b","metric_details":{"license":"Proprietary","variance":9.954601414854002},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-sonnet-5-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e5245b2a316a5d00f48b888afa647512c7592a2aaae626996b51a5055437fda8","result":{"confidence_high":1545.5455114812187,"confidence_low":1533.1777806607997,"sample_count":14130,"score":1539.361646071009,"score_display":"1539","source_stated_rank":40},"run_fingerprint":"f0dd44878985507f7bf97d75d46102cd1d78a7a30f82dddaaf9c40f0e30ceb87","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c2ecec4951050e7335f990ed2f34b2f866fbabeaf496832b98b9ecc7a450d78a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-5_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"6e68bb87053fe0fabee96aefbe9bd955e0d5c920a38db88d3ed81aa79e2aa253","result":{"confidence_high":1543.87,"confidence_low":1529.57,"sample_count":9296,"score":1536.72,"score_display":"1536.72","source_stated_rank":40},"run_fingerprint":"58af5aea4f56ea9b26a20d63eb7733fe2a196d6e2bc05e7e48a719913ce2afbb","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"993c21b6a950b34643a19b8bbb503f309b152c66eaf77c490ce3c1fea064045e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"19f25720664102c965f733bca8baeea6b5ec87314087432ac558fde76487ce3e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.78,"score_display":"68.8","source_stated_rank":35},"run_fingerprint":"b7d99baf391abc639a47526cfdd7367125bcdc3749f41cbfd6abeacaf21e0dc2","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"78535b3bd6731a62abec9a42d9cf24c5f3044529e8433c2e8f9da7c5645529e1","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.534595,"date_is_proxy":true,"latency_seconds":635.043,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.085},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-5"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"a6b11f8dd40b30e30b8a60dc9e445a898b43c89794149c590779efa9b88b910b","result":{"confidence_high":78.6186,"confidence_low":70.44539999999999,"sample_count":null,"score":74.532,"score_display":"74.5","source_stated_rank":18},"run_fingerprint":"0cc386a8a3a3b3fc21f556da0e2eac911bcd1e3709925a8c582dadc85907a607","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2d6999462b27faa0fb1f0946537d026fb261598b9b6db5f726595019ae43e999","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · terminalbenchV21"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-sonnet-5","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"6e275161d0c53ec43957d69f63ca58e095b2d2ca9d692d811efa90197f7462ae","result":{"confidence_high":87.415259,"confidence_low":71.107358,"sample_count":89,"score":80.524345,"score_display":"80.5","source_stated_rank":null},"run_fingerprint":"233fb1021866bce0171a54443d57401938fdc8e5490e6346d3cbc48e5d5c5fec","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7e5521731bbf19df6ddbd40496a5e1006c3df158a6f034e8c146cbf98dc6bc4d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · terminalbenchV21"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"claude-sonnet-5-non-reasoning","upstream_model_label":"Claude Sonnet 5 (Non-reasoning, High Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"be00c58eb001e291b6c6cd5fa40e6284c68f5607a9d18fcb36bbf58a15a9189a","result":{"confidence_high":83.071885,"confidence_low":65.397767,"sample_count":89,"score":75.280899,"score_display":"75.3","source_stated_rank":null},"run_fingerprint":"deadc65911c6c2e6362478ef63c9efed260da530d93deda01054bb6a8c1bdecd","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6bca2a2ee2ca5f4a6d2be994","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T21:39:34.661007+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T21:39:34.661007+00:00","status":"fresh"},"measurement_id":"5fc4d38e5cbf9f6e67dd7365b8c4bcfda39b00dc2fcde76413c9ee60d8ebd0ad","metric_details":{"accuracy_stderr":1.5,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:07:50.908645+00:00","total_cost_usd":6890.54,"total_tokens":17948790860},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"Anthropic","url":"https://www.anthropic.com"},"agent_url":"https://claude.com/claude-code","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"a945badd-d54b-4c45-965d-aa35a042c27e","model_release_date":"2026-06-30","model_url":"https://www.anthropic.com/news/claude-sonnet-5","n_trials":370,"source_row_date":"2026-07-12"},"run_count":5,"scaffold":"Claude Code","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"25f28bd29e55217425ef1850eacad7c52d96511c3d9abb79c78a195ddc7fad33","result":{"confidence_high":24.371986,"confidence_low":8.303088,"sample_count":74,"score":14.59,"score_display":"14.59","source_stated_rank":10},"run_fingerprint":"4448b4dcbe691b01f052cebcc4c29723835582278a9910e49bb06f3a8ec555c6","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8da390c0c0f8da7938f604c9471bcfb4c4d2664d852c28a80f882a3b515a7afa","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · tauBanking"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"c0f6bd8ea4403758b061764ef6a9fc9c9d6ebf7a4aa362b3eff4ce5cc27e35da","result":{"confidence_high":47.254974,"confidence_low":28.350333,"sample_count":97,"score":37.319588,"score_display":"37.3","source_stated_rank":null},"run_fingerprint":"a877fd190cf44fefa7c7a4673c2941bc5f813389c9f04a2c4dba406fb952cfac","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b9e075b96bd7ef3b6429efae582b28b6967b996c8759dff91e80816b237cb980","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-06-30","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · tauBanking"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-5-non-reasoning","upstream_model_label":"Claude Sonnet 5 (Non-reasoning, High Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"61b3c750aa67e3b40f695fa30e621dab4fa6794fbf624b7a318e75c903957b4a","result":{"confidence_high":24.192609,"confidence_low":9.763218,"sample_count":97,"score":15.670103,"score_display":"15.7","source_stated_rank":null},"run_fingerprint":"3603966f0b62dfa6ddb5f5154257815cd712b4384a684c321c6ffd9d06eeb879","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6bca2a2ee2ca5f4a6d2be994","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d3b3244c90c3aa3b7644de55dcd91708342cf8d98d9c77c8f59573cf0baf0b2e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.674","mean_standard_error":"4.5","model_release_date":"2026-06-30","notes":null,"standard_error_points":550.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 5","source_effort":null,"source_model_version":"claude-sonnet-5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"14a88483065ac3fcfc7e9d99405ba73e6e2611d954422d95e711aaed3eb4dac3","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":54.50000000000001,"score_display":"54.5","source_stated_rank":17},"run_fingerprint":"d22effac45ccf303f70c0b7dd522288b3ac13f03c281c93e111c43d0a9398ff6","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f8d65b9fce47c16194d9f6c1","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d9036b351d9b2807f154b00a99ec254149bc195c4e5389844e1218cf2cc35ab5","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":32467},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Claude Sonnet 5 (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"dfc4c621e43f0d2b4358c5bc93923febf19a838f630554d96caf851d3f580ef4","result":{"confidence_high":0.05979717597318822,"confidence_low":0.028172889685818808,"sample_count":3612898,"score":0.043985032829503515,"score_display":"0.0440","source_stated_rank":13},"run_fingerprint":"11c8918feea7585aaf6d14669a229c7b0b1f1ebeaaf6fecf3bdbe6de5874dd21","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8f05cdf4b5d12f81ad9036bc14001e25d1db733ffc675b64326fac9a7ea0c28f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-30","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 5","source_effort":null,"source_model_version":"claude-sonnet-5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"7145326ec3a37c3c686b6b29841d280a2481e6f2ccf82d77a0769f9153e1b663","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":6377.701666666665,"score_display":"6377.70","source_stated_rank":15},"run_fingerprint":"c41be51177bc5fa00c749d5ca07a0491e338568516b6b1310560b9638cf650e9","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_145e1c54c865c8920fcac152","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ea502c48634922bb13ef72ae8451be8122c37e34f885b585a84fc140a6ad1d71","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"e55e4959ff419aa91a5a01cc68fd329c20cf0ced59935ff256aed24362ebc44c","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1449.2,"score_display":"1449","source_stated_rank":null},"run_fingerprint":"30f67bce2bacd69a33a6c265c1b7b269aee189121d6f95a384757841e0d13a03","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Max Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_289738e1411638bab936a9b2","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1f5556f71d73bba60ef14127a8f1f5ed8e79deee78b5c496c5fe5caf2a480333","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-xhigh","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"31d7b3687dda25be8f79eff262180bf2a8fc8c7bdeefc8682f1fd84be584e836","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1344.4,"score_display":"1344","source_stated_rank":null},"run_fingerprint":"b06b07d7e884661f6ad6d2b7d6d97a1348a6c00502c18322452f57269b70bf13","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_24d6255a6b90676ceab86615","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3f51e843cf0e434b5c5e44eb1f195b1bec61bd894c010545b98190496e5bc792","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-high","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, High Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"35a70d05400e13dca95932fbb2ce39ccbd9db431e7d21a52d6d396cce918c93f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1245.72,"score_display":"1246","source_stated_rank":null},"run_fingerprint":"85dc5074471d7b0b9d8cc144a5fc5507e46096b54c861cd40575aef3927669ca","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, High Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":true,"normalized_effort":"high","profile_id":"profile_37845fc871b455ab26fa8ccb","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f1fbbab870bad34df3cb27a140b3f2b4ddf836d621101d36483e7b7fe992a8f2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-medium","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Medium Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"f7bc669f27701c4b05d467b96482c1607d60a43ce017a51e9b861d242e0218df","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1144.76,"score_display":"1145","source_stated_rank":null},"run_fingerprint":"b0500ada67bcfa451c4d26cc0d43b040b28386b038e4ecc752f58d780e79ab9a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Medium Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_017911fce805da3d633adae6","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3dedc2b2b1397523934a633839e9107d830408c7a767033516f909164272af58","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-30","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-low","upstream_model_label":"Claude Sonnet 5 (Adaptive Reasoning, Low Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"3435a956ed849190f24ae638d1c5f10a05cfb81e7ea5eb15b596ca0a41f4610a","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1058.55,"score_display":"1059","source_stated_rank":null},"run_fingerprint":"cb7d03644190e6303d1b277fa3f4dbc87e90639861aa48289b85d68a1974365b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Adaptive Reasoning, Low Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e672ac0437f08612f7384fba","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"af29aee5711a026e5b1d501a1fbbe3e032621fd691ea258424925994d1a9c583","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-06-30","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · gdpval"},"model":{"id":"anthropic/claude-sonnet-5","name":"Claude Sonnet 5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-5-non-reasoning","upstream_model_label":"Claude Sonnet 5 (Non-reasoning, High Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"13582627e3d2d85d97bafeb7a16cd50d8b9b19f1cb8a11ef6aa427916a19f5e0","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1200.1,"score_display":"1200","source_stated_rank":null},"run_fingerprint":"36684a3e0356e0178b700894d6e2a05945d9a0f50b2eff379edfa8a84a4a4350","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 5 (Non-reasoning, High Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6bca2a2ee2ca5f4a6d2be994","provider_config":{"provider_default_effort":"high","provider_default_evidence_url":"https://platform.claude.com/docs/en/build-with-claude/effort","source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4f84a25ddf0c36e050864790df2ea4fb83ce79eadf735f76aa06e95433bd0ade","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-08-14"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GLM-5.3; model release: 2026-08-14","source_accessibility":"API access","source_model_name":"GLM-5.3","source_model_release_date":"2026-08-14","source_model_versions":["glm-5.3_max","glm-5.3_unknown"],"source_reasoning_efforts":["max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e0a15915752ede821648a0292aef14763074ed382182331591fc1b6ef00fbe4e","result":{"confidence_high":158.34,"confidence_low":153.65,"sample_count":null,"score":155.76,"score_display":"155.76","source_stated_rank":24},"run_fingerprint":"9051353781284915252a966e90da138c2e06c3aa3724845c20a9d4a9086aa3ec","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_35a37f6bd374e637bd675128","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6052dedf0df06b49e3f1db82d97279ee7601a9328537cb3a1e7c7faca9194244","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Max) · intelligenceIndex"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3","upstream_model_label":"GLM-5.3 (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"3fc231306e79cd5532cc6b9acff0c46cb2420eefbe31490bb5fa1a42a144b8be","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.777392,"score_display":"44.8","source_stated_rank":null},"run_fingerprint":"7bc3636f8a10740b3c9f1558bc45bf1913355983059aa6e1150697d3d97fea39","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4b7a0ffd88f0fad6dfb4caf69e4d487d95675b18407b5bd7850bddd9442a9aaa","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Low) · intelligenceIndex"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-low","upstream_model_label":"GLM-5.3 (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"a664a472b5d675cfbef11fab00a497076eb675d114369957beb13ed48163c572","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.299008,"score_display":"34.3","source_stated_rank":null},"run_fingerprint":"ec1c5b23cb1cf71434b03ea04862c9238ccfaa6a9bc13b3f02f9fbb67f96e32a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b22dd146e2f8620d258e3c97","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c97ea78101b41b3ecfd43ac9d9a42a2de5b9cedaa35f57b15820e5308b23b81b","metric_details":{"category_scores":{"Agentic Coding":60.909,"Coding":78.95,"Data Analysis":70.24433333333333,"IF":69.304,"Language":79.85633333333332,"Mathematics":87.898,"Reasoning":85.803}},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.13780952380952,"score_display":"76.14","source_stated_rank":32},"run_fingerprint":"108a9515aca36c4915ae1c3dd6d5a61e726d917fd531b7152d8163a10e03b2b0","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_59194fc28b0dddffb241f8c7","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9f3238b6788f5bb38888bd19c94930bff8e613ac140ea543a64f4370aef7acfc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":"With tools.","num_parameters":753329940480,"pull_request":null,"result_file_url":"https://huggingface.co/zai-org/GLM-5.3/resolve/main/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/zai-org/GLM-5.3"}},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.5,"score_display":"62.5","source_stated_rank":2},"run_fingerprint":"2c985d876db9a7ac78e592d7b4408f31a161596342fec771fd06630e89f90c03","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1e27a40ca87a4086efefafd6","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"720b7733f2d0612502a358279ad212ad2089dc3fd7a0b16bddc13abcc0adad88","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Max) · hle"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3","upstream_model_label":"GLM-5.3 (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"aca9514f8a570f5bf9c5a2a2762b8d859ac13614a8e04f2a6225279fefe14f46","result":{"confidence_high":44.357478,"confidence_low":40.192731,"sample_count":2158,"score":42.261353,"score_display":"42.3","source_stated_rank":null},"run_fingerprint":"c41997c1a067ba5641e4d8b400bf1432bbd0bbe9b9c3cace7d131b880b72c65f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ddc35eb9cce3fcd0f46f74c944db6073e5318ffaafc96e9c07f03803b56199f8","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Low) · hle"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-low","upstream_model_label":"GLM-5.3 (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8e33fd94250589e951e75ca3d1ea215348e5b716bc57fc62cc37484d4ea4099c","result":{"confidence_high":38.615824,"confidence_low":34.555198,"sample_count":2158,"score":36.561631,"score_display":"36.6","source_stated_rank":null},"run_fingerprint":"72c597356fe441421d79b2e196d942c7535a16dbc19bfebe686247269eed854b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b22dd146e2f8620d258e3c97","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T17:00:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T17:00:06.000Z","first_observed_at":"2026-08-31T22:06:34Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b8aa3ad724fdc30e8483ce089d3c51d2bee60c18d9a08e8f90159887d1ee2239","metric_details":{"best_score_across_scorers":"0.41","model_release_date":"2026-08-14","stderr":1.556091713692166},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"TY8R8FV7zCwByEipkqyg2P","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTY8R8FV7zCwByEipkqyg2P.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/TY8R8FV7zCwByEipkqyg2P.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"be976fb14801559fcbcf0a3c3d24dd863d841e7563f53649aa5b2d9b1f5d7f06","result":{"confidence_high":44.04993975883664,"confidence_low":37.95006024116336,"sample_count":1000,"score":41.0,"score_display":"41.0","source_stated_rank":44},"run_fingerprint":"a3f5521612881462ed49311d13e944fbd87846cefaf774b9c1a27e5dc0e07bdf","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"686fecaae43d995a1d7462c49d78d8225e89e8c490c6936710887612f78dacb6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.055878,"date_is_proxy":true,"latency_seconds":210.7,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.851},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":88.889,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"max","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3","zero_shot_accuracy":87.374},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"bbee015919fa0d8b8f3896daa07f06a671e771ba78115b618b3df03f8477cf3c","result":{"confidence_high":91.75996,"confidence_low":84.50404,"sample_count":396,"score":88.132,"score_display":"88.13","source_stated_rank":35},"run_fingerprint":"c97bf4e773a529b937c33ae7df42d94e14d63c88ed7fd41cc17a986cbe50293d","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3548420ab9750689726c93fbd55c6b52a4177253ca768670bacbce99720b4bc7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Max) · gpqa"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3","upstream_model_label":"GLM-5.3 (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"36aca4ed44e28a4adbf3567c1eb4f8c123ab0cd4bd76b05ce2a43b3e9920f3ba","result":{"confidence_high":94.807658,"confidence_low":87.038701,"sample_count":198,"score":91.717172,"score_display":"91.7","source_stated_rank":null},"run_fingerprint":"4c485d98266087f6702ad3e513835bd30f130c376ba0f480d88219a2d29373e4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Max) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":41,"at":"2026-08-24T19:44:18.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-24T19:44:18.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"57e428ef7c2e0382a9d3dfd102e026839e88b93deeb44dccab8ded65779bfb61","metric_details":{"best_score_across_scorers":"0.9090909090909091","model_release_date":"2026-08-14","stderr":1.5851578202142853},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dLDBahYBTGRNjbsXA5pyr3","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"26f350141917fb4d2756e8aa4ec9dc0ff22c3ddd72c1dd6b00d9b86c4e4aed73","result":{"confidence_high":94.01600023671091,"confidence_low":87.8021815814709,"sample_count":null,"score":90.9090909090909,"score_display":"90.9","source_stated_rank":34},"run_fingerprint":"325b65d0b50d797a7f04d244802ef2c2190c7dec5930681adab3c066f9bfca59","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2b34c1bb4f8d1b8d807ab005d93bee5849ebdc122b85f799ff56cff3883587ef","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.016639,"date_is_proxy":true,"latency_seconds":61.783,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.335},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"fd642deb3a40d45c0ec7ae9e916614602b7e89a6f0d66f00cf563b075929144e","result":{"confidence_high":87.4266,"confidence_low":86.1134,"sample_count":null,"score":86.77,"score_display":"86.8","source_stated_rank":38},"run_fingerprint":"f1e07503c71916153a1671fddbe7454d33bf2f7d332b0baa9e99f38746e783b4","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":41,"at":"2026-08-24T19:44:18.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-24T19:44:18.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c8bacebe93faf3d57f1d8168960005ae4fede51ffca778c700a7febb1783a146","metric_details":{"best_score_across_scorers":"0.9111111111111111","model_release_date":"2026-08-14","stderr":3.454754040013868},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"goZR5yGxZic9TZ23ezCL3w","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4e8cb847c4372d821424c8e84d071e4d9eddfa79832e97f96e4beb381d508ce9","result":{"confidence_high":97.8824290295383,"confidence_low":84.33979319268393,"sample_count":45,"score":91.11111111111111,"score_display":"91.1","source_stated_rank":65},"run_fingerprint":"7bad120447a679ef28419f18b938ede12658dc8454a482b184c74b6bedac5730","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":41,"at":"2026-08-25T14:20:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-25T14:20:20.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e4225f7c519277942df071f3faa8c6fd135fb49993d3dcea65b36a00f7a5b5f3","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.6877192982456141","leaderboard_private_problem_count":285,"model_release_date":"2026-08-14","public_example_count":10,"stderr":2.7499133473298816},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"iXvqUzwxAiA3Rg3dwnutFs","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GLM-5.3 (max)","thinking":null,"upstream_model_id":"glm-5.3_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"baa8b449e7584a3a04da45802bc5a09fb052cc1712837ea86e91e067754922a0","result":{"confidence_high":74.16175998532798,"confidence_low":63.38209966379485,"sample_count":285,"score":68.77192982456141,"score_display":"68.8","source_stated_rank":25},"run_fingerprint":"95d08cab9309c5ad66ab7faa6e8747cf9df5c288823c56b137d0c9de69f1c5ba","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":41,"at":"2026-08-25T14:20:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-25T14:20:20.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bff3d23135487a0f9eae7be5138282b12bb529358a55ac39bce3825c6525efcc","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.2926829268292683","leaderboard_private_problem_count":41,"model_release_date":"2026-08-14","public_example_count":2,"stderr":7.194088392074452},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"4S3ZrJDkwh4gmGrmJQdvzR","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GLM-5.3 (max)","thinking":null,"upstream_model_id":"glm-5.3_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"7bb80b144278a2bde3d74797f51778f83a6e103dec6bf21308a03e696749ae13","result":{"confidence_high":43.36870593139275,"confidence_low":15.167879434460902,"sample_count":41,"score":29.268292682926827,"score_display":"29.3","source_stated_rank":34},"run_fingerprint":"ac462762605f0ca1649f24609dfb90d10c9877f9152550190f0b46c6eb2e7dd2","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"feb834b63c3acf1f4f28c301c1ecee6e6a2477bbfeaac7131bbf2a5800c0eb96","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.3 (max)","source_effort":null,"source_model_version":"glm-5.3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c725f5a9eed07a383dd0c4650d29de0fa16104150f68d7d0efaeb12e708befeb","result":{"confidence_high":29.779854,"confidence_low":11.673638,"sample_count":71,"score":19.1428571428571,"score_display":"19.1","source_stated_rank":57},"run_fingerprint":"5384f2f69126304f7e0affc7c72af93e1ad58b07f8c3094227d33ba3d992b8d0","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bda755bd4fec04892341184a462882d87c178a0db08ac0080d39f08ed58b47b0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Max) · critpt"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3","upstream_model_label":"GLM-5.3 (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"9fe2e74742cfbce0883c098d8017b789c3e3053d62731fa09dde3f52eb64dc4a","result":{"confidence_high":29.864288,"confidence_low":11.632104,"sample_count":70,"score":19.142857,"score_display":"19.1","source_stated_rank":null},"run_fingerprint":"280df89fe5d864c3424ccf45c0ee8aed7af6ef818530ed86fa7caf3189a41dfb","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"13f431b521652a47b80c75a5ab01621e1dc174873ac2fe84e54591bb8a201bc0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Low) · critpt"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-low","upstream_model_label":"GLM-5.3 (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b6f30b00ad0dd4d05152ee1acf46ac528bdb431fa4730e8de090084381e746c3","result":{"confidence_high":24.670423,"confidence_low":8.158769,"sample_count":70,"score":14.571429,"score_display":"14.6","source_stated_rank":null},"run_fingerprint":"74eeaaa0ab81c5e04c41b0cb22411cddd382d8d213d2abd2f4c3f562b6d48056","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b22dd146e2f8620d258e3c97","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":41,"at":"2026-08-24T19:44:18.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-24T19:44:18.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6b16e42275cc1191fe4a2e3039f3a39fdbc3c09cac3c438cd64e05b3a69006f2","metric_details":{"best_score_across_scorers":"0.21","model_release_date":"2026-08-14","stderr":4.093601807403323},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"9RfS3yHmzmP3azirghceFL","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"951d5a2b2ddf69c33226b7e7221a399b30929a35641fbacad07d4a0544b58ae3","result":{"confidence_high":29.023459542510516,"confidence_low":12.976540457489486,"sample_count":100,"score":21.0,"score_display":"21.0","source_stated_rank":80},"run_fingerprint":"07f9f28c243ce99c49e8e28535dce94f7cc0f50f7654cef231d05eae408f776d","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d324acaf90cf11d82d3d6bbb051f1a4b5724c1543099b2448257f96cb2939560","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.338311,"date_is_proxy":true,"latency_seconds":840.831,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.938},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"8167195cfbd030761ac72f2a48207d70096fd661e646df99e3de147f8248adeb","result":{"confidence_high":97.23848000000001,"confidence_low":93.56152,"sample_count":500,"score":95.4,"score_display":"95.4","source_stated_rank":5},"run_fingerprint":"e701ea1f62ba0d3a29f8da65ffa5121776fa2a5c140e4456478bdcea8e20f7a4","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"1ede9aebd761ccea03f90e5410016eb80c9c6caabda5bbd03d56bfccf473e609","metric_details":{"ci_attempted":451,"ci_half_points":3.0186512715587,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":311,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":124.47228381374723,"mean_cache_tokens":13106008.833702883,"mean_cost_usd":3.9933584893126386,"mean_duration_seconds":2116.1784804168515,"mean_input_tokens":13272344.266075388,"mean_output_tokens":80435.60975609756,"median_agent_steps":114.0,"median_cost_usd":3.1315945999999992,"median_duration_seconds":1842.82226,"median_input_tokens":10137783.0,"median_output_tokens":74953.0,"median_output_tokens_to_pass":74814.0,"median_peak_context_tokens":144729.0,"n_attempted":451,"n_passed":311,"n_tasks_attempted":113,"n_tasks_passed_any":99,"pass_at_4_points":87.61061946902655,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_glm_5_3_max","source_model_version":"glm-5-3","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"76a7e4fafabe3bbe3146b6c402e8d8e6542f2b5390eb800eeb16eb81377c59dd","result":{"confidence_high":71.97652266845449,"confidence_low":65.93922012533709,"sample_count":451,"score":68.95787139689578,"score_display":"69.0","source_stated_rank":16},"run_fingerprint":"41f75462411b9de0ecb0ba1f07a859047626474a2948fe71336e06b9b58323a7","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4045807453e3c06945ef49df6b5013d56fff8f275f43906fe6db960cf320b71f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"124.47228381374723","mean_cost_usd":"3.9933584893126386","mean_output_tokens":"80435.60975609756","model_release_date":"2026-08-14","notes":null,"pass_4":"0.8761061946902655","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5-3 (max)","source_effort":"max","source_model_version":"glm-5.3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"d3d281ba16234e1592c3f98eb86c11b64de61ff9761dcb321b41ebd66b90279b","result":{"confidence_high":71.97652266845448,"confidence_low":65.93922012533709,"sample_count":113,"score":68.95787139689578,"score_display":"69.0","source_stated_rank":16},"run_fingerprint":"824793d346c83d8c0acdb386a77c14b2e5a1550b1a6572632714df16c5bc019d","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"543e751a32041868317ee266c0a64a3ded38cbebfdf94fb7384f061ab285f06e","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM 5.3","source_effort":"max","source_model_version":"glm-5.3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"chisel","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"efc0f59f353073dfaf952df51d9124e1b208b5b06ff4efd1ecda47e34f72b7e3","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":40.14,"score_display":"40.1","source_stated_rank":22},"run_fingerprint":"91901497ba18b0b5a3ae772dc562b82d797dbe44561b3e9035dbadea43270d96","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4e66c9b69bb949d5d6d018ecf4d05a6ee04423211f7bb7b1b7bf273ec257aaf6","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":7.67087,"date_is_proxy":true,"latency_seconds":6374.465,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":7.476},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"4a549da4ba3ae0bc1f615ee33488b9b3082472e12adc49e053cbc779ac078212","result":{"confidence_high":83.09696,"confidence_low":53.79104,"sample_count":null,"score":68.444,"score_display":"68.4","source_stated_rank":12},"run_fingerprint":"55a18b31c9d5abdf641918036d63ae261892e33f69a450fa216bc31ba558047b","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6a22b917986f162cbfdca653ad0983248ead06a531569972aca974e5de0c0c4c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.069397,"date_is_proxy":true,"latency_seconds":246.75,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.065},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"max","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"fd9e10b9c6fa09d7c5b99cda04c69d1718448f705ffb408d5adb5ffdd18a384e","result":{"confidence_high":82.6224,"confidence_low":78.4476,"sample_count":null,"score":80.535,"score_display":"80.5","source_stated_rank":68},"run_fingerprint":"b0ec13b42f8c3ce6908ae245004649b9221c755275a39200f4cbf75412444012","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f6f50bde07a2f4af93972eea88f312ae1f76070001092941aeae48ba9fccf82e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.3 (max)","source_effort":null,"source_model_version":"glm-5.3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2323af851cae82609800cbb343d10f682841bc30ea08a5d9c8f098d351e4b914","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.0277777777778,"score_display":"59.0","source_stated_rank":12},"run_fingerprint":"5563016d55e101e4a45d366d5aaa903b61db1ad7dfee7a7b43ec5fd76035ce88","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4863b8bdc1479db87fd98eb9ec1e12a8fa4636c2ddd109571667f6c15128f250","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Max) · scicode"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3","upstream_model_label":"GLM-5.3 (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"839b46188a456abf10a06b08b0cbcc95d67a26c1944c6abe980b580d1165a26d","result":{"confidence_high":64.552484,"confidence_low":53.265401,"sample_count":288,"score":59.027778,"score_display":"59.0","source_stated_rank":null},"run_fingerprint":"5c8de31b888121d3a6dcd51b4e8f714f3f97326a886b2f4dccaa57a5c2b72987","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"70b680c7fe453a4dadcdd095b9537ff08106e6e92deb2a0a9497c2768c2543e9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Low) · scicode"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-low","upstream_model_label":"GLM-5.3 (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"72cba80385b955c911ec5f3e9c7b4bd03f42dfc91c0259bb1a5fc56ca1391c7e","result":{"confidence_high":47.782915,"confidence_low":36.455111,"sample_count":288,"score":42.013889,"score_display":"42.0","source_stated_rank":null},"run_fingerprint":"67c70c065f06c48d2d3f67090abd07f403c3851f206f8c1b60228dba5f08ca48","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b22dd146e2f8620d258e3c97","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bcc6715b4729443657d56961d90e13e79f315f8f0693b83d45ef92d73493d4ca","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.284001","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5.3-high","source_effort":null,"source_model_version":"glm-5.3_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"daa02bfb234208aad8d174c45bd774f76c33f7031a76c9bcbe564a266bbe1a38","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1317.4,"score_display":"1317.40","source_stated_rank":26},"run_fingerprint":"f081e2747bfb2834242fb93523820c3c37548f589b0a0545e31bad7f5b2db087","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7b670530e689719e8ea1db1f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"97145d82cee2ab8d144e6f1f34fc73de33e013b2c82f28e5ca949944b47d0035","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":12.450185,"date_is_proxy":true,"latency_seconds":3847.197,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.94},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":131072,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"5de263e31f5d9338a945bc60beea2014a37d2dd217c9735200ad9e9765c91df5","result":{"confidence_high":85.8474,"confidence_low":70.4026,"sample_count":null,"score":78.125,"score_display":"78.1","source_stated_rank":25},"run_fingerprint":"d0201290fbe80ea9a7d9fe60a10d13bc5982bb6d5cd2765a13a3efefec4aaff9","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2635ccf39cc559dc9c39d78332831c9430f1e062dd4d81ba94cf70f41ff1710a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.3_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"b752c7181f86a93bba823695b13debc8da98e8017c428b45df5c621594cf6a9f","result":{"confidence_high":1625.29,"confidence_low":1603.61,"sample_count":3725,"score":1614.45,"score_display":"1614.45","source_stated_rank":18},"run_fingerprint":"ade05eca006db21ed8b0eb83f634c7e4877e3d4e884b07a23512da3a644abbfc","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:22Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5ce6985f159ff9915d88970a94fcaaaf7369ecb0fc5d4797d883a000ac09b3c7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.3_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"d235c007ddac6805eeaf070ba3e06dcefb7f1a6c7cd41114fa61624a508e20ee","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.4,"score_display":"75.4","source_stated_rank":29},"run_fingerprint":"4272d41f17ade46e17fc44f16393ab158a78e3a27b5d7e9afb2656c4c1810eef","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9a894423e0ca1ba428524e8a1ded9bca81ecebd0d02d18f22b617d42c7724106","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.30672,"date_is_proxy":true,"latency_seconds":786.604,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.982},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"0dd939721a55fda3f510c944b4c26e44a511eb68ccd2691a7d31734e814a49c9","result":{"confidence_high":75.42072,"confidence_low":67.65128,"sample_count":null,"score":71.536,"score_display":"71.5","source_stated_rank":26},"run_fingerprint":"eab1f46bef8064d3ec0d8a93b9b73f215bb6b3fc2a89b971c91493fa62d6b44a","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3cfc3f718fceacad2719f3db80978334465a5bb33dbb4202fbe1cf570643d48a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.3 (max) · terminalbenchV21"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-3","upstream_model_label":"GLM-5.3 (max)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"0c382f97bc02c0d2cf0cc04a85ae3eadd42541c1d84a50037b2057d0631d8bcf","result":{"confidence_high":90.100094,"confidence_low":74.885142,"sample_count":89,"score":83.895131,"score_display":"83.9","source_stated_rank":null},"run_fingerprint":"b75c67c479e0daa712aee209ebd6a1f11f464f17bb1d411ebc5fe2b1f70a0155","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (max) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":47,"at":"2026-08-19T03:41:59.486409+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-19T03:41:59.486409+00:00","status":"fresh"},"measurement_id":"937f41faeead6c07f404fe8c0532b34f770fce9974408fd6e8f5ea900c1ad89e","metric_details":{"accuracy_stderr":1.46,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:07:01.521097+00:00","total_cost_usd":1771.02,"total_tokens":5603245878},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"Anthropic","url":"https://www.anthropic.com"},"agent_url":"https://claude.com/claude-code","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"d5152814-6bf2-473f-be57-b1908ed480a9","model_release_date":"2026-08-14","model_url":"https://z.ai/blog/glm-5.3","n_trials":370,"source_row_date":"2026-08-18"},"run_count":5,"scaffold":"Claude Code","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"53ff18eba7ace83101286cb02f5020ab9cc5293ebd9dda6d00301c775e303b7d","result":{"confidence_high":43.73241,"confidence_low":22.861801,"sample_count":74,"score":32.43,"score_display":"32.43","source_stated_rank":4},"run_fingerprint":"bbad5dac0df5b973c869634b5dfe9cc7bd9a7c39e5a03b4408ca0fea978a19ed","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"00c886b8937d69def21db9cb4306b93e03ed6bd6e760bb1c0227a3533510dfd0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","source_locator":"Leaderboard models table · GLM-5.3 (Max) · tauBanking"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3","upstream_model_label":"GLM-5.3 (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"294efe3fa535ec695cd6bda375688f73b76adbe1b9f77e9aca4468815cf06f71","result":{"confidence_high":60.056336,"confidence_low":40.538655,"sample_count":97,"score":50.309278,"score_display":"50.3","source_stated_rank":null},"run_fingerprint":"6b2951c933d874b2e0043b48038f407f6f84cdbfdce24662967aeab6c2c1cad2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Max) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":17,"at":"2026-09-17T14:59:54Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-17T14:59:54Z","status":"fresh"},"measurement_id":"65c80970f29cc211851160bc7a20aa2c2b638c2a1b21a740e2560a1211c171c3","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.15,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=GLM 5.3; createdAt=2026-09-17T14:59:54Z","source_row_created_at":"2026-09-17T14:59:54Z","task_pass_coverage_threshold_percent":75},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"GLM 5.3","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"ecc5b6422c2e2e3429cabe318a44da1978fca6fa339fe247f0f534ae84a9240c","result":{"confidence_high":86.35000000000001,"confidence_low":82.05,"sample_count":1000,"score":84.2,"score_display":"84.2","source_stated_rank":2},"run_fingerprint":"f9f5e0d48349986b7c64b66fb7513132edca812c21bdc27c40be7edd17df83bf","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=GLM 5.3; createdAt=2026-09-17T14:59:54Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c742d73fd887bd38522e03c0","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7d74738e7a3bfbc54abda4758c96eddd6ac1e23e7d437b8035f0da0b7c21c93c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.662","mean_standard_error":"4.3","model_release_date":"2026-08-14","notes":null,"standard_error_points":509.99999999999994,"upstream_source_url":null},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.3","source_effort":null,"source_model_version":"glm-5.3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"8e8a234bf10a298637422da5a4ae9b86500d0848c73b6b7a5c8bc68beebeb6c8","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":56.60000000000001,"score_display":"56.6","source_stated_rank":14},"run_fingerprint":"3cb22f95096208cf518fa277668ffad83e51f8cacc2e8abc67d2d337db74cb67","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_628c201ab8499b38d223a8c1","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cbe00295f62d448685068d9b32f7f665b0107d11eb972a4f82d67574694848b6","metric_details":{"confidence_level":0.95,"license":"MIT","session_count":86384},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GLM 5.3 (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"965f80307cb843e0a82e38c10261e5244c2cb851442fd889a227ac889fa42548","result":{"confidence_high":0.030424576683927985,"confidence_low":0.016740103879208115,"sample_count":9155315,"score":0.02358234028156805,"score_display":"0.0236","source_stated_rank":23},"run_fingerprint":"7588c9a7ecd1b778ed46cd9f386613bb1e266d09c096e239675a7bc3dad6a258","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ffc218f72cb8df4cb6bc39286c7f69ca1b4c9bdc13cce24459ad2d76f7fafa98","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.3","source_effort":null,"source_model_version":"glm-5.3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"460f57bc4832717e13967632b0bcbe5ca1bb1a7e4fe87aca0ed24ae8a70042dd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":8163.609999999998,"score_display":"8163.61","source_stated_rank":9},"run_fingerprint":"7f18edb53202c845377a2587d03981610360d17f3116749fbee73bdbbf3685e0","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c07ed0ced91381c8fb6c9cb5","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-01T22:31:28Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:28Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"65f518d5cbc96b729dfebd51df770a2997117a0d1842d55371657fa6c4f1f8fb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","reported_confidence_interval":"-21 / +21","source_locator":"Leaderboard models table · GLM-5.3 (Max) · gdpval"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"glm-5-3","upstream_model_label":"GLM-5.3 (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"52fa29974da54411a3f8a69161f3a9b5e034cace9f0b13d9711c98b8c9a284e3","result":{"confidence_high":1676.96,"confidence_low":1635.59,"sample_count":220,"score":1656.28,"score_display":"1656","source_stated_rank":null},"run_fingerprint":"d2a92c36f8f439a48822d459b716fa5681efbdb70f5357eb709a13bb66cc332e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c9ddbc6765550e54adee760a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c1c5cdf2b5cecf169a47fde5ddf6548444bcb497a2ba9e877e5db5cdd76e479b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-18","reported_confidence_interval":"-18 / +18","source_locator":"Leaderboard models table · GLM-5.3 (Low) · gdpval"},"model":{"id":"zai/glm-5-3","name":"GLM 5.3","provider":"Z.ai","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"glm-5-3-low","upstream_model_label":"GLM-5.3 (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"b796fc1cdab4cb270d3a458d5d28f2a4eecf2f5647041f2cec9f034679991784","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1304.83,"score_display":"1305","source_stated_rank":null},"run_fingerprint":"4c74c7482a8b918ae621b19ad47f3713d56489af7bf7ef711e32c94193ce5453","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3 (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b22dd146e2f8620d258e3c97","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"74e5b53ed09aba240f615a6b28722f2e4dbaeec92774709999f661ebef839e34","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-12-11"},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.2 Pro; model release: 2025-12-11","source_accessibility":"API access","source_model_name":"GPT-5.2 Pro","source_model_release_date":"2025-12-11","source_model_versions":["gpt-5.2-pro-2025-12-11","gpt-5.2-pro-2025-12-11_high","gpt-5.2-pro-2025-12-11_xhigh"],"source_reasoning_efforts":["high","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0c2a729e69b2c56a53936191b44197ae28c64588d6bcc9f7a5db8da51099a4a2","result":{"confidence_high":158.23,"confidence_low":153.03,"sample_count":null,"score":155.41,"score_display":"155.41","source_stated_rank":25},"run_fingerprint":"9bf2987408845abadd611d8bcf0164eab93333799c87ffa90c6e7ac64f4fead6","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fd182e68dd93da34ca6196d3","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-13T02:37:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-13T02:37:55.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"46cd4af44a7639b093c2cfe8bbf1b277863048337843c07d16c79709456f83dd","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.74","leaderboard_private_problem_count":285,"model_release_date":"2025-12-11","public_example_count":10,"stderr":2.6349537704451795},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"dMJenf5xYWu7Pwn2U6w4dW","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.2 Pro","thinking":null,"upstream_model_id":"gpt-5.2-pro-2025-12-11_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"8c74b08a97c0383941cf14c84ca5818f0c07fa819359022155073dcd71799c6c","result":{"confidence_high":79.16450939007255,"confidence_low":68.83549060992745,"sample_count":285,"score":74.0,"score_display":"74.0","source_stated_rank":21},"run_fingerprint":"3c3c49f6399d07cb032aa2ff3b358ed8e59a4470ff5764c65be1341082b503f6","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ba440b37e9b02e11c009b3","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d2df4afcfd2f5d0ddde4c5cd10db26f5bcefaa25e527c38738bbd7a2962d04dc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":15.721,"model_release_date":"2025-12-11T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-2-pro-2025-12-11-thinking","model_type":"CoT","upstream_model_id":"gpt-5-2-pro-2025-12-11-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2f85e349795acf0d68e5ae22a7718ee829611fcb56986f06c43659bda088f931","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.16,"score_display":"54","source_stated_rank":89},"run_fingerprint":"e4f9a971480cfe7668d56492187bd15d3d666a563373d344a432d16520ab990c","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_2fa48c668af270933840111e","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"75aca8c2f3dbcb39925472598f1225238ee60846fdb06da3e42412734c382c7c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":8.9928,"model_release_date":"2025-12-11T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-2-pro-2025-12-11-thinking","model_type":"CoT","upstream_model_id":"gpt-5-2-pro-2025-12-11-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"19a676046d0b05291d21cdb8fa1ac14a1e46938c8de90d55a6af01f7227edfcb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":38.47,"score_display":"38","source_stated_rank":106},"run_fingerprint":"4042e871230eff2c2af2339c7072c04819600da3eb9320bbe6a3895795225dfb","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_832125207d2aab00c427e506","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"65d3921c7087c33c718fc21af67ac8dd3ebf33bd0b465e2b4ffec3f621001ad0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"15.721","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 Pro (High)","source_effort":null,"source_model_version":"gpt-5.2-pro-2025-12-11_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c9311398201ee58921fcca599bece10b5517ffc8280aebfc780157da7d9470d0","result":{"confidence_high":59.236166,"confidence_low":48.995988,"sample_count":360,"score":54.16,"score_display":"54.2","source_stated_rank":91},"run_fingerprint":"440e81868dd6ef14705dd2c80bbc2532bac563ec760909ec59332b4f00f391bb","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_2fa48c668af270933840111e","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"29a641c07280fce2681905b91d85c4cfa5b1cf29745e55d1af57c7bf3a948b46","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"8.9928","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 Pro (Medium)","source_effort":null,"source_model_version":"gpt-5.2-pro-2025-12-11_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"2bdd46ee9dc9fbaa1bafedf07225d8b06f4ab47e686514e5a2a68eb33ccf9470","result":{"confidence_high":43.592466,"confidence_low":33.591011,"sample_count":360,"score":38.47,"score_display":"38.5","source_stated_rank":107},"run_fingerprint":"e6877618bb94ae2c3736e4fa17053492dbe0299149a50eea7de53a97349dc71e","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_832125207d2aab00c427e506","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-13T02:37:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-13T02:37:55.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ea9b7ae5bf69626fc3a93215c5799ef7e76492c97f5cbaf8b465ccbd4c2f5ff4","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.46","leaderboard_private_problem_count":41,"model_release_date":"2025-12-11","public_example_count":2,"stderr":7.789619230571372},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"oALBfwNc7dcNSDJSSBJXYe","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.2 Pro","thinking":null,"upstream_model_id":"gpt-5.2-pro-2025-12-11_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"97480455bd56f5d849116086cfed4deff8497a6a66c499469ade8dfd31e3d42a","result":{"confidence_high":61.26765369191989,"confidence_low":30.73234630808011,"sample_count":41,"score":46.0,"score_display":"46.0","source_stated_rank":25},"run_fingerprint":"44f90bc2db393981bb3f9f0f05d91c8c0d82617c4106f6f6f21acf51940fbb18","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ba440b37e9b02e11c009b3","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"28b1d6d26e19b241c84061e1c167bfe17b3aa8220ca72b885556af266cc3b621","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":"5.2 Pro xHigh\n","upstream_source_url":"https://simple-bench.com/"},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-pro-2025-12-11","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"641e8efe24c5d7264e335e4f1b1372035aaed643f9a70ef1a1b65a599e3f5883","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.4,"score_display":"57.4","source_stated_rank":38},"run_fingerprint":"a1bef4e4d008ceb431f02fb011d32fd4d7f222361e09218f06c822f200685ec8","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f84193b5a06372776da088aa","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"35b2b679ce1a357990435edf37a57a5697b29230c6b44354c990c00c6f0785df","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"11.6542","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 Pro (X-High)","source_effort":null,"source_model_version":"gpt-5.2-pro-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"254d25a57b3cc806c85d610086ba76a6ff094450b15629846f2053fd880cae07","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.5,"score_display":"90.5","source_stated_rank":70},"run_fingerprint":"b3ece5e6fe8c76d1c11241d22160f88e4e35928bcb20e9358b9bed9f7c09a7ff","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ba440b37e9b02e11c009b3","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3326a765eb14b22e12bd82679ea92f02c7c0e01afadc610550e6a7a0f12cfe51","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 Pro (xHigh)","source_effort":null,"source_model_version":"gpt-5.2-pro-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"367a84ed0c37b48277b8688ab6753fefae71dc27a362d0fdfbb614a3e550b125","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.5,"score_display":"90.5","source_stated_rank":70},"run_fingerprint":"3f1542e82d8323fbe755daa6c82eebc25192276ef331dcbc7827cc1ef9fb937a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ba440b37e9b02e11c009b3","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dba667a29f349a96956b54b72ecac25e58ca1cf6b37225e4e8eb65e142961991","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 Pro (XHigh)","source_effort":null,"source_model_version":"gpt-5.2-pro-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"21606065809f6b3fa5ed2af1493d1763e969298faf60be4bb477a5d55746e16a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.5,"score_display":"90.5","source_stated_rank":70},"run_fingerprint":"2fb7bf3deda3ce73d36df8e2225111ec80deca76c639e0a8e0b90f4d11fed8de","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ba440b37e9b02e11c009b3","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"673c2947bf69ffc585fe5b0ed09941f67abd47bbdbf399a70d712d41b0d1056b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"5.8694","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 Pro (High)","source_effort":null,"source_model_version":"gpt-5.2-pro-2025-12-11_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"496a4957592e7aca85c42e322ea3859083e45956831dc7db8b599ecc9cf60ce3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.67,"score_display":"85.7","source_stated_rank":102},"run_fingerprint":"63a028f82cbc1311dc0f640608dd563b38b79cf0754b33e54d5f02dd10198170","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_2fa48c668af270933840111e","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4dc659c93b559af684d2a47981171b2d9480d13147c6fc6ee8f9216daa953a86","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"3.9774","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-2-pro","name":"GPT 5.2 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 Pro (Medium)","source_effort":null,"source_model_version":"gpt-5.2-pro-2025-12-11_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"861acdb845b0937fa5f894a351a1339c29f4abe0f39a1b3ad825f60744e048a4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.17000000000002,"score_display":"81.2","source_stated_rank":110},"run_fingerprint":"1f0020232f88ec27b9028e31974e8912e4eba4dab0c46dabe6a451d6e7473e2a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_832125207d2aab00c427e506","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c65147722d825c4fbb9dcf10200106d8f94d00fa47587b153ac0d93ec74b9926","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-08-13"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: DeepSeek V4 Pro 0813; model release: 2026-08-13","source_accessibility":"Open weights (unrestricted)","source_model_name":"DeepSeek V4 Pro 0813","source_model_release_date":"2026-08-13","source_model_versions":["deepseek-v4-pro-0813_high","deepseek-v4-pro-0813_low","deepseek-v4-pro-0813_max","deepseek-v4-pro-0813_unknown"],"source_reasoning_efforts":["low","high","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7c91b0bccb7ec34079fb57969502fe1404fff7c1696dee9f7b6a3f451f88f8fe","result":{"confidence_high":157.56,"confidence_low":153.66,"sample_count":null,"score":155.38,"score_display":"155.38","source_stated_rank":26},"run_fingerprint":"b4471bab7a1f62461bde386966aef29850ee0a7025297c217244a13214faf46a","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c09f67f64bed3296f4de97af","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d91893342cc92e80b7c1eea180b0138df07f8f484b047e99384388f2b1924546","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · intelligenceIndex"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro","upstream_model_label":"DeepSeek V4 Pro 0813 (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"42125f128e354b05bc3f42362814aba6aae89a3f6eaf7c6d72cbc920c3de9ce6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.996779,"score_display":"36.0","source_stated_rank":null},"run_fingerprint":"83779c93243a3474691f04715371ca4d770bd6f65010f87bca9ed3ca4dd9bbd0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6211f15fc3d0b788fa3765755f198531243a74bf833dbf445ad31205a818d4a3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · intelligenceIndex"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro-non-reasoning","upstream_model_label":"DeepSeek V4 Pro 0813 (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"1400f1990d312fca6f14946e3588457da147d3e6ed6caf8005c6daaf840745bc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":20.36051,"score_display":"20.4","source_stated_rank":null},"run_fingerprint":"509a63bb96e857bbcfceec76a535b9c60602ac572f509a32c8fae273c3d5807c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3152edaac5f1fd3d1ecc5bbe","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"430a4b8ba5866e37efeb9dbfb0d86fa9c138a79b00e2fb522fb4a1b448a24c10","metric_details":{"license":"MIT","variance":11.431538160059207},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1470.9052763771506,"confidence_low":1457.6517812759992,"sample_count":9780,"score":1464.278528826575,"score_display":"1464","source_stated_rank":56},"run_fingerprint":"bd38735a786225fcdf20cea20fe6a98ba191681974769adacd0038a3738ebe66","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30663e63c085ce7e9e35d6eb","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"7a7e238ba1a93d1cbb75dca9472a079487b82a2bd910a73d76b5089bdb8846b5","metric_details":{"category_scores":{"Agentic Coding":54.949666666666666,"Coding":77.1585,"Data Analysis":79.24333333333334,"IF":67.69999999999999,"Language":82.074,"Mathematics":95.08624999999999,"Reasoning":85.84125}},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.43614285714285,"score_display":"77.44","source_stated_rank":23},"run_fingerprint":"4c1c898544624d2dd217204f3226fe943e6e5947e7b501922c30ef2e708bb578","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2a6ad306680e2f1f01edb742","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_1f4bfa5486eb4deec4942449","split_or_window":"provider-report-no-tools","unit":"percent","version":"HLE original"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":53,"at":"2026-08-13","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"fresh"},"measurement_id":"76e6fabeb085a834f0e7d3bac34867becf3e12b44dd61b68598b8985a5b9b128","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table · HLE (wo / w tools)"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"DeepSeek model-card HLE comparison; tool status reported; exact dataset revision and judge not restated","reasoning_effort":"reported","result_published_at":"2026-08-13","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"75c0bf5cc62a5e8a5b5579767a3b94349eb76845210b6368a4b0ed833f857d74","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.7,"score_display":"42.7","source_stated_rank":null},"run_fingerprint":"7d9433fb208028a63a95bc36a97f4e7b0326f9b492979994c13e4b00ba7f4aa1","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table · HLE (wo / w tools)","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a6b20b4d1f7b9f4ae3e96950","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":53,"at":"2026-08-13","basis":"evaluation_at","evaluated_at":"2026-08-13","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5de69c0a6f2c8afe2a9f57ccbd04009f7c5fe4d52a3c3aa013261181ccd18596","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/deepseek-v4-pro-0813","source_model":"DeepSeek-V4-Pro-0813"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2eeb032adb228a68e3d849e1cfabad6772cb7c130e03437b1f7ebd21247466c1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.7,"score_display":"42.7","source_stated_rank":10},"run_fingerprint":"8cadcd8d8d4625b637d73abb666ae68b082bb79c7924ef676a4ed8bec6a9d1de","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5131b7118ecddf36f42391c2","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5dc5214f4902e17be96d77ff","split_or_window":"provider-report-with-tools","unit":"percent","version":"HLE original"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":53,"at":"2026-08-13","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"fresh"},"measurement_id":"892772a6d28d5e96243e674eb856f443bf958c1b837bc87b7a54ee65f2d1b3a6","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table · HLE (wo / w tools)"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"DeepSeek model-card HLE comparison; tool status reported; exact dataset revision and judge not restated","reasoning_effort":"reported","result_published_at":"2026-08-13","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"tools enabled (unspecified)"},"run_count":null,"scaffold":null,"tools_allowed":"tools enabled (unspecified)"},"protocol_fingerprint":"3dfdb03494d6480a497ab33309ee25ff0f4a2c28c7d241d15e57d35ed521a75a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.0,"score_display":"60.0","source_stated_rank":null},"run_fingerprint":"1ce001162125b48ff3504dd50f9c0d0ded76cb876b637af1d7a6982fafd8cc51","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table · HLE (wo / w tools)","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a6b20b4d1f7b9f4ae3e96950","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8bdb489c51575cf17efdc3b09cb877c8541ed7ade25692643d4a9274fd4c8405","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · hle"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro","upstream_model_label":"DeepSeek V4 Pro 0813 (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"427f5d34b1a5781339b1ef7983c6c9329622bc11191364525543c535a99f97a2","result":{"confidence_high":43.09961,"confidence_low":38.95273,"sample_count":2158,"score":41.010195,"score_display":"41.0","source_stated_rank":null},"run_fingerprint":"ec929299d31e9dffb86f9bc96be59f7e628f7c523256563e13f0733152d451f1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4f3795321589a97f848e4439557d9195a743132a4d6c298b8111ebbdc4c5223e","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · hle"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro-non-reasoning","upstream_model_label":"DeepSeek V4 Pro 0813 (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7324d894f66d0b503021085c215c022c0848158a374b6fdad0b375169e37ac36","result":{"confidence_high":11.933109,"confidence_low":9.337718,"sample_count":2158,"score":10.565338,"score_display":"10.6","source_stated_rank":null},"run_fingerprint":"fae7d8d554740939ec053b685b8a5a6c6c414657405c82b6cb5bc8b439575cd4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3152edaac5f1fd3d1ecc5bbe","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:30:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:30:20.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"82c4687dc2d55cfd628bd24eb7f668c878df2690f5415f82e872e7de4447d394","metric_details":{"best_score_across_scorers":"0.5290581162324649","model_release_date":"2026-08-13","stderr":1.5808394665706598},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"2xfFf6KZAp2gUWYhHGkFgs","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2xfFf6KZAp2gUWYhHGkFgs.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/2xfFf6KZAp2gUWYhHGkFgs.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"fb389f5e6ce6839d1fb8f0a8eab180a25968c0d0c926ccf278768bcba9be7348","result":{"confidence_high":56.004256977724985,"confidence_low":49.807366268768,"sample_count":1000,"score":52.90581162324649,"score_display":"52.9","source_stated_rank":21},"run_fingerprint":"00720ace26f5d1908050fb72f194cb22a685e5d370d08c2a565fe5499c235980","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b74f7637f6f06177a44fa718f7e6803ecfdf5551148382a0146ceac46acd31a9","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.065819,"date_is_proxy":true,"latency_seconds":279.412,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.02},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":91.919,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":384000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"max","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro-0813","zero_shot_accuracy":92.929},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"d941b57a59dc6da0b399b02484a04ac79163c0a35b08fc3852e37bef25aedc83","result":{"confidence_high":96.3832,"confidence_low":88.46480000000001,"sample_count":396,"score":92.424,"score_display":"92.42","source_stated_rank":16},"run_fingerprint":"05fbff415e02113dcec14fc987035344d4622557c106d6c80bb35f92c49ed052","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7992aa9fd6e0b2c770128940e0420aab841b0110f5ba6d336030230416c756de","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · gpqa"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro","upstream_model_label":"DeepSeek V4 Pro 0813 (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"01333b448374542e3eee176bbd9604dfae64d61c98cad66d58f0a4a38e118d21","result":{"confidence_high":95.664893,"confidence_low":88.361393,"sample_count":198,"score":92.828283,"score_display":"92.8","source_stated_rank":null},"run_fingerprint":"af7653c7bdfeb17743346027e2041064c7962fcc3db9815f0e3c644b7d0d7e1f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":47,"at":"2026-08-18T21:03:23.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-18T21:03:23.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c8d55ecb90a8ef46b6ce9dcd0adb4dc32b6fe8c8d055292ad3e25a93a4d5c81a","metric_details":{"best_score_across_scorers":"0.9166666666666666","model_release_date":"2026-08-13","stderr":1.5242573629934337},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"nEJGtXGN588zynPkynZkxz","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"0eb08ba8af29e44154b79c580693b959ccdc4aed7545dfad6c85b5fa243dc79c","result":{"confidence_high":94.65421109813379,"confidence_low":88.67912223519953,"sample_count":null,"score":91.66666666666666,"score_display":"91.7","source_stated_rank":29},"run_fingerprint":"f01be11a11badf69070ecb1ed8f6ecaebd32b251ab612a092ebcdd4cc4858d40","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1c231903ccb205050324f70ad5421641d900ff64080d2ec295c8e39ab632d622","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.01283,"date_is_proxy":true,"latency_seconds":59.624,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.341},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":384000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro-0813"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ba8a74160fe9d7cd11b1b27c38b947323635afb082cbd44111fd539ead9e12dc","result":{"confidence_high":87.64036,"confidence_low":86.30363999999999,"sample_count":null,"score":86.972,"score_display":"87.0","source_stated_rank":36},"run_fingerprint":"70a1989805a208ecebb3c0a46302b5cd42c14b905ef67f5d4f65b3707655991a","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":47,"at":"2026-08-18T21:03:23.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-18T21:03:23.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ec4b9acca411e48888e5686b6a994eeda813146c433f9c8ae8aea3cbabd2d00f","metric_details":{"best_score_across_scorers":"0.9861111111111112","model_release_date":"2026-08-13","stderr":0.990980650017605},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4HVvM9ZWKKPLQPcC8WYYMT","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"93edc8b8be7acc7253a111ad5e77411d3b0836ac0cb76c70fedc517ebc5c97cd","result":{"confidence_high":100.0,"confidence_low":96.6687890370766,"sample_count":45,"score":98.61111111111111,"score_display":"98.6","source_stated_rank":20},"run_fingerprint":"2b0f9dbff758d5b931fac55cc0441693f037a8d5256e1ab368ed0fef643ccfe3","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":47,"at":"2026-08-19T13:35:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-19T13:35:48.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8689c4cd35b2b7e016eb6fc3a5d061c4f7c76e0a2b6ca610c211b4c7d54952b2","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.6456140350877193","leaderboard_private_problem_count":285,"model_release_date":"2026-08-13","public_example_count":10,"stderr":2.8383475205435627},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"b6yWmeFgsWeEiG9Qzs3NSf","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fb6yWmeFgsWeEiG9Qzs3NSf.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/b6yWmeFgsWeEiG9Qzs3NSf.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"DeepSeek V4 Pro 0813 (max)","thinking":null,"upstream_model_id":"deepseek-v4-pro-0813_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"3a7dbc6bd2b185e89d1634f1e9b098cfb3a5c5a2eb6a502f5c17c352164289e9","result":{"confidence_high":70.12456464903731,"confidence_low":58.99824236850655,"sample_count":285,"score":64.56140350877193,"score_display":"64.6","source_stated_rank":32},"run_fingerprint":"ab31f670384f3756c5675595abe3e15516de49ba66d7baeca61b3e622cdc6cc0","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2a2415cbf08afd7506bb4cd4852ef73e2d93ac843825f9759dea0ee48b143d74","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.5976109150000003,"model_release_date":"2026-08-13T00:00:00.000Z","results_url":"https://arcprize.org/results/deepseek-v4-pro-0813"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"deepseek-v4-pro-0813","model_type":"CoT","upstream_model_id":"deepseek-v4-pro-0813-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f54664a069cd3a263e222b62884cb95777c40556c3db565b8f8b542e48ca7239","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.25000000000001,"score_display":"61","source_stated_rank":73},"run_fingerprint":"9c2a33041336575eddf7cfdcd1ee24de4bc34c2227cac427ab98120c865d27dc","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b4c4e2bd98c5fdeaa161e44c64439a1a3d88125b3cbbef4234cf3130190abc54","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.5969432040000002,"model_release_date":"2026-08-13T00:00:00.000Z","results_url":"https://arcprize.org/results/deepseek-v4-pro-0813"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"deepseek-v4-pro-0813","model_type":"CoT","upstream_model_id":"deepseek-v4-pro-0813-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"08679f8e5dfc9e254aa28825ed8de9a6929f74348f3609448efa68cbb40ade09","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.72222222222221,"score_display":"60","source_stated_rank":78},"run_fingerprint":"633ff21cceef9f783b3ec48fdadef0388348b5d64c679dd96ad24d88da9b5c16","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30663e63c085ce7e9e35d6eb","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f00fe39603dcef3d1c108e876079aa2785be6e43878ac33ec30bdb8567e21962","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.37025881200000005,"model_release_date":"2026-08-13T00:00:00.000Z","results_url":"https://arcprize.org/results/deepseek-v4-pro-0813"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"deepseek-v4-pro-0813","model_type":"CoT","upstream_model_id":"deepseek-v4-pro-0813-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b107eec600b4f6eebcccf9b035926e564b1bdc03c2f85d941bff7ff1dedd72f3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.25,"score_display":"56","source_stated_rank":85},"run_fingerprint":"5807388c88a9e246fa87b55b0974bd5bbf8714bb852b55e4be46dd5b9e026558","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4b1820067ed08e7cf0ee5988","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e8869147658aeb2ee97be74d572661432fc52023b77cd7dfe523985cc6e6865d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.040291977000000014,"model_release_date":"2026-08-13T00:00:00.000Z","results_url":"https://arcprize.org/results/deepseek-v4-pro-0813"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"deepseek-v4-pro-0813","model_type":"CoT","upstream_model_id":"deepseek-v4-pro-0813-none"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"81197bf8101c755a988451fe45517f13db2653bb6228fbf3ef016b1263a20b48","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.8333333333333334,"score_display":"1","source_stated_rank":226},"run_fingerprint":"96d6735d3fb89f3e4a2bdfc35e7fed9b830cdaf9559abf444004497db8cc681f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3152edaac5f1fd3d1ecc5bbe","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"88e116700975b0ec35a52610ace67a619d70acd08e5d18ff2b751d80d3f92735","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (Max)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"bf411ebc66e005fbe463769b183a07bf72e237b6d17a51788c81c737b6cc85e0","result":{"confidence_high":66.138603,"confidence_low":56.123832,"sample_count":360,"score":61.25000000000001,"score_display":"61.3","source_stated_rank":76},"run_fingerprint":"f722cbfe88bfbe0e37d201a91eeb4dd353e3b52a7da393b641c80230d3494c1b","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2bc8f80b9fb4327d34513527da8c0b0d8440df87447da8c10bc12dcbaeba3b64","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (High)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ad8efa4d61d324751973660c23a3920060b8f57670664f808a05940a5afb0fb6","result":{"confidence_high":64.660269,"confidence_low":54.578872,"sample_count":360,"score":59.72222222222221,"score_display":"59.7","source_stated_rank":81},"run_fingerprint":"a699ce4069d9ee2ffd280058d5acbba99d35b243459018a41cd06b478e56edea","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30663e63c085ce7e9e35d6eb","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8b4cb6ebd9db30e88686535ea8200315d942da1b2ae3962ce2ad4bb586f9acb5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (Low)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3b3871d5e4147555060977b6c924652bc467005d3a8c3975ebcfb989739b92a5","result":{"confidence_high":61.281854,"confidence_low":51.086165,"sample_count":360,"score":56.25,"score_display":"56.2","source_stated_rank":87},"run_fingerprint":"985985b78ff534346c93817815638fc9d4b8755c0efed6ab01b7718db86d42c2","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4b1820067ed08e7cf0ee5988","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c2ecf06be587eca52e7f9a90e18739bcb04543de54bdedb6eacc4a748aec8dd7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (None)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5c501c694102bcef967cf16a44dcb58a31fdb81a3c4bc808941913d356976628","result":{"confidence_high":2.421113,"confidence_low":0.2838,"sample_count":360,"score":0.8333333333333334,"score_display":"0.8","source_stated_rank":218},"run_fingerprint":"6ac63a0f14f42fb82a2f353cf684aa59338bf2e87335671559165434db715e0a","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3152edaac5f1fd3d1ecc5bbe","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":47,"at":"2026-08-19T13:35:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-19T13:35:48.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d40cb000430c187407fc3d55aa419e3cd44e021460a0892a493f28224f80047c","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.2682926829268293","leaderboard_private_problem_count":41,"model_release_date":"2026-08-13","public_example_count":2,"stderr":7.005564203095157},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"a4uV8HMMBAcHXLVSmZfBgX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fa4uV8HMMBAcHXLVSmZfBgX.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/a4uV8HMMBAcHXLVSmZfBgX.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"DeepSeek V4 Pro 0813 (max)","thinking":null,"upstream_model_id":"deepseek-v4-pro-0813_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"65a81bc4e77c19398496f8304279491bf76b327f284447513082d40883dded3c","result":{"confidence_high":40.56017413074944,"confidence_low":13.09836245461642,"sample_count":41,"score":26.82926829268293,"score_display":"26.8","source_stated_rank":37},"run_fingerprint":"a2459a06cd55a29ac98e5382cfe48c4279f8488fe9f2438f74b75e2a3b19d80b","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"47373d618484bb77750aba31ea2b9486b7593437fb12114da716e9b7c9fda42e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a331bef24c0429c2f7ed978f81bbb8916531a8fb624004045b3a4a1673088e68","result":{"confidence_high":28.500355,"confidence_low":10.784749,"sample_count":71,"score":18.0,"score_display":"18.0","source_stated_rank":60},"run_fingerprint":"66d4ef101478995e5edab2f2f78b84f5708a6094e3d3dc8f6e7cf761dd787fa3","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0f0b60da2ded4e6cfa9551b28a2a6261a477e3df7ac205f35d14061612b375ce","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · critpt"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro","upstream_model_label":"DeepSeek V4 Pro 0813 (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"dbe3966b96e73684f7ba8937c9bb793b79058a7d96f1b9b958a3b080d69c9694","result":{"confidence_high":28.584452,"confidence_low":10.74514,"sample_count":70,"score":18.0,"score_display":"18.0","source_stated_rank":null},"run_fingerprint":"ab608b13b7283698fcb6ff927d663fc945e78eb91beebb330b6422304fb088f6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fbc8e17bec30c70abbc30c09cf1ce575bb47cf01826addf899a5bff2dc525965","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · critpt"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro-non-reasoning","upstream_model_label":"DeepSeek V4 Pro 0813 (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8649646c6b501697acf2910f894ff26887b812dd82f2b65043cd2046cc7436e2","result":{"confidence_high":5.202487,"confidence_low":0.0,"sample_count":70,"score":0.0,"score_display":"0.0","source_stated_rank":null},"run_fingerprint":"1595bce2ff71463e57b137686a1c6d3b05763a8ca6a67c5ebaae655e09a1240c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3152edaac5f1fd3d1ecc5bbe","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":47,"at":"2026-08-18T21:03:23.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-18T21:03:23.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9e36310d4b1c465e83583b2095a3265728ef7f484956cda8b4a443866da53252","metric_details":{"best_score_across_scorers":"0.47","model_release_date":"2026-08-13","stderr":5.016135580465917},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"EyKecrMhk7Yx25e5HGW43i","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"90ccb7af0069c6df60ab5b0421549de2ad6bd217ffb7340edb6e6fd9476fd9cf","result":{"confidence_high":56.831625737713196,"confidence_low":37.168374262286804,"sample_count":100,"score":47.0,"score_display":"47.0","source_stated_rank":14},"run_fingerprint":"0c4fce8c7c83cd9f95aeb897d00e57f9f40a073269536d67bceda8fee8a5ee31","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7312d01a0a88b2c64e7e8ca5849a1708b35422648f7ddadb5acf9ccab58189a7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (Max)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"20ec2e9d0c96a3ad110511dd3bc6664dc17452fcbd0bf5682947eab4b984831e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.0,"score_display":"90.0","source_stated_rank":75},"run_fingerprint":"50a29acfa04f98ff0920bfd65f0ab6366e08f285818bb5a3fb37f4604a089090","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1807168120ea8c637dd7b91b69412ca1199d7bb3bd7f0ae941d527bae872bfdc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (High)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"25441df600b12d188c427bf670fef47ff4610a101ab4dcfe6fadfc74c5847764","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.16666666666666,"score_display":"87.2","source_stated_rank":88},"run_fingerprint":"83a3d6bb1c68d94bc8ee8a363abffc85db8138d164c00968a888c6556891cb3c","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30663e63c085ce7e9e35d6eb","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"624154f3dc91354853ddc4f5e142632b5b37fc58ddcfacf76073e75912d0784c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (Low)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"1710c818580be795a5942c02f5e82da94dc7af33ac0d17375ac46bf8a73fe133","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.5,"score_display":"90.5","source_stated_rank":70},"run_fingerprint":"f112418c1868a60ac51bdf6f30a827cfc3993dbf9a01dd3ecccbf36a5b177be7","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4b1820067ed08e7cf0ee5988","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"85069419b6b3e3ee649e4e60607cc41b1b62b31e5e99e44f3233e1cb4ab00c18","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (None)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"8f646c9227a6dc53877cd29f52439e30f72ae0892169347f0fc6f13862308b88","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.0,"score_display":"13.0","source_stated_rank":243},"run_fingerprint":"c19678f0658e8c0f897b58f181d9d025adf10ff48b8d9c9a061fc109145da92c","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3152edaac5f1fd3d1ecc5bbe","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c0a5c18dbcdea30915bdac973ea8fa77366539bc1157159b96cacb116dd1b29b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.103104,"date_is_proxy":true,"latency_seconds":240.413,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.834},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":384000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro-0813"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"837f729b56e1b17d2e695490f09951ee3ec1481f1c8370a2676d28c931a97da6","result":{"confidence_high":98.03464000000001,"confidence_low":94.76536,"sample_count":500,"score":96.4,"score_display":"96.4","source_stated_rank":2},"run_fingerprint":"358de4817369a066acb845767fe2c614ecb8d0cc0d4f804ea1d204893aeab57a","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"349e5d00efde812e5331d5c935b2108e05e282c904435e1c318f220575244d9f","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813","source_effort":"high","source_model_version":"deepseek-v4-pro-0813_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"chisel","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"c2d8a6033488029e021ec03b8455b148e517875e5207235c823a01af6a2b5c7f","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":28.549999999999997,"score_display":"28.5","source_stated_rank":28},"run_fingerprint":"160de85062265380987d426fc5e7c985795962ae59f4a32ed8fe46805d3f0b0d","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30663e63c085ce7e9e35d6eb","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1f742d7b2a3e924eaf266527989bc593c704f3d60a8142f62e3033ea0be03f7d","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.151589,"date_is_proxy":true,"latency_seconds":4045.972,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.4},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":384000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro-0813"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"6d0063b09a9275be7fa2e1ff9b3380e41c0d683481981881b93a53135a4e7400","result":{"confidence_high":56.315,"confidence_low":46.907,"sample_count":null,"score":51.611,"score_display":"51.6","source_stated_rank":23},"run_fingerprint":"5d08c55990f5e3ef5d1fe0cd9e8a8512e13d3cb7541e1b90ec91502ce8b16876","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6d11f1e575e323b4518c1c8dc792b3efaed6a0f30927a2b7c8d61deac5288b0c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.063876,"date_is_proxy":true,"latency_seconds":273.233,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.958},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"max","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro-0813"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"1fe37d0e8fb740e6476aae322503d20bbc539f80445affa24e87cdfec59685e5","result":{"confidence_high":89.40268,"confidence_low":85.64732000000001,"sample_count":null,"score":87.525,"score_display":"87.5","source_stated_rank":13},"run_fingerprint":"de13e4a201a5a2f52142fef10a2adca67359a5977193686d597289d3de153579","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8768cc3bd217e21c98f4434d86079ac88f7fab01198cfab5a3b787c0a5848946","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro 0813 (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fbb4bf23f93064419d5c25f7d0e863cf278e68b867f184f6a4003d0affa6773f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.0416666666667,"score_display":"51.0","source_stated_rank":81},"run_fingerprint":"5c8e1f20aa346fc8fff918995889efea8daa0b6f8fb7c9da2aff05cc9c317a95","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cfb359efeb1c174b2e05bf5ce12e2e938962c9fdccb1f93198f70c3dd9784478","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · scicode"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro","upstream_model_label":"DeepSeek V4 Pro 0813 (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b2cd00d85601081669499b1cbe2c6587736e80faeaa77a5ae1beae5aa196d21d","result":{"confidence_high":56.763299,"confidence_low":45.292612,"sample_count":288,"score":51.041667,"score_display":"51.0","source_stated_rank":null},"run_fingerprint":"68d6e2fa2ddc6ac0bd11e7a3dcb4f38ee57489b376105bafa27bb3d753234a89","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f41604682269d4238fb65016b5623ad8b6cb658f7f7d24bffcd63b9ff1bba811","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · scicode"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro-non-reasoning","upstream_model_label":"DeepSeek V4 Pro 0813 (Non-reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"465f076fad55859458b59a9f4bc8472e7bc8dee07e66a4e36ed111f069a798b4","result":{"confidence_high":45.683704,"confidence_low":34.442502,"sample_count":288,"score":39.930556,"score_display":"39.9","source_stated_rank":null},"run_fingerprint":"5c7c982ef840220d3f96e77598864913a2a1cca7524cfaffc4d705906d06439d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3152edaac5f1fd3d1ecc5bbe","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"936f84d35ffe69be30a0ae7dc65695635ea778a24c775bcb871840cc16b8f078","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.072764","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"deepseek-v4-pro-0813-max","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"6891d233e4e51d5e32e85f40613aa84ec70db2d12f8707e09e131f6272670e8f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1403.15,"score_display":"1403.15","source_stated_rank":22},"run_fingerprint":"98250532bdf5f97a86bf25df99372a77657e0298bdf3159fb5c8d3254eaf95e3","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"88e2c7bb715356cf54a1a88b6059c6190654ff816b608573183290f1cbb5f3ad","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.356023,"date_is_proxy":true,"latency_seconds":3982.6,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.148},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":384000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro-0813"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"f3786aca42e7cdaa937178fbf4352f2524027d6d3bc124f0e0dd490f5af2ff48","result":{"confidence_high":88.46708,"confidence_low":76.12692,"sample_count":null,"score":82.297,"score_display":"82.3","source_stated_rank":18},"run_fingerprint":"a3c978b80b69169a48be2fffaf1cab1bcfbac0fefccf13a4e9719ed5024a75d6","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"10c4fedf8d6d33e1852a03aff73f2554d11e5324116e23ce9b282382bc19b08a","metric_details":{"license":"MIT","variance":22.689282274006477},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"deepseek-v4-pro-high-20260813","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"84796124120dab58cdbd84b22483beec2f92bbdc104ee8e49254f7ab759fd2f9","result":{"confidence_high":1591.9747265083515,"confidence_low":1573.3028285301918,"sample_count":4882,"score":1582.6387775192716,"score_display":"1583","source_stated_rank":30},"run_fingerprint":"f2f5868d3defbe9bb26c14f0da08e98edd53667767711f467b9d2a8bca5aa6bf","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30663e63c085ce7e9e35d6eb","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"34c0ccae61ccda54faee52fd1918db9358c7671ca448828dfd8e061ab2c5ba15","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-pro-0813_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"98927c2fea90376510dc021ef8db9b5a3253777e3c60eb08f1eb9b4ef26b549c","result":{"confidence_high":1590.48,"confidence_low":1570.59,"sample_count":4258,"score":1580.53,"score_display":"1580.53","source_stated_rank":25},"run_fingerprint":"d2909127f5ee3dad306da5ad46e41161505bee8726810ce0ccb8765bf4d64523","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30663e63c085ce7e9e35d6eb","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7c47959ae0df05e37ac1bea900e1c9c99278aeb8fd4b2d50c7f934c5d766d6ce","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-13","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-pro-0813_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"876d95f59e5e7ca1a61549d0c9418536be0db88d61f34ea6708f45f226b33e68","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.22,"score_display":"66.2","source_stated_rank":39},"run_fingerprint":"6e73b1cfbdf5f933f94683ba03f1345e5d5e86bdcd1ce0e08a9c235e5e95c8cc","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5229ec7dd080f68460ba73294a0a10bd8995bf7b3b9873a9a45867bffa3fb34c","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.199883,"date_is_proxy":true,"latency_seconds":862.374,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.498},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":384000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro-0813"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"8f922f1ede0eca72e4302c7a9a81fbecf27e657c5d8bb911cc6da346aee7e05c","result":{"confidence_high":57.61808,"confidence_low":51.745920000000005,"sample_count":null,"score":54.682,"score_display":"54.7","source_stated_rank":51},"run_fingerprint":"61e3a78f420934803231a7a36ec8d78361a5a3159f4f36844ef0613725628aaf","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"513d95afcbdf7229b719ed3806beec89a8d3b5569df4c5e5426a1134fcc595a5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-13","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Reasoning, Max Effort) · terminalbenchV21"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-pro","upstream_model_label":"DeepSeek V4 Pro 0813 (Reasoning, Max Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"4d48a50b6d13ffb0149b940c949caffdefe6ec3fba1fd452571de57ae9235c57","result":{"confidence_high":85.885313,"confidence_low":69.046958,"sample_count":89,"score":78.651685,"score_display":"78.7","source_stated_rank":null},"run_fingerprint":"58502815fcfb5ba0e63cc1929e641cd15b8f5a5bd2fe2b67e65cd4c68ee75256","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Reasoning, Max Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_2190ac806074ce92366c1f75","split_or_window":"deepseek-harness-minimal","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":53,"at":"2026-08-13","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"fresh"},"measurement_id":"284cc9c2b65dc83cde3f535b58de57e1950b20e1781dfb0221cd56bcd34beb93","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table and note 1 · Terminal Bench 2.1"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"max reasoning effort; temperature 1.0; top_p 0.95; minimal DeepSeek Harness","reasoning_effort":"max","result_published_at":"2026-08-13","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"terminal agent toolset"},"run_count":null,"scaffold":"DeepSeek Harness minimal mode","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"d2761572c128446d0c01731770c26ae8eda5e78171869288b5252a4580202e8f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.9,"score_display":"87.9","source_stated_rank":null},"run_fingerprint":"ab91620b4ff077d6b0ce365535894acaa0e9d3ced7c699f42083b29c74141412","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table and note 1 · Terminal Bench 2.1","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ed39629eec2a08ed59b55fe39075cfb4e2211644cdcb130f2c1cdd401d92f685","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-13","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · tauBanking"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-pro","upstream_model_label":"DeepSeek V4 Pro 0813 (Max)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"3663822e954121fe22c798fe50922795f5009491be30bc9d593005659b4f4a9b","result":{"confidence_high":49.537596,"confidence_low":30.430989,"sample_count":97,"score":39.587629,"score_display":"39.6","source_stated_rank":null},"run_fingerprint":"abbd5ae0705788a27262563c9bc2e225efe74de8fdb2acaa7892f072e50540f8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"414f99330c7bfb5eb8aff926f3296638ab3c0012c174b47ecfc4a5ab8da5d183","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.623","mean_standard_error":"4.9","model_release_date":"2026-08-13","notes":null,"standard_error_points":560.0,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek-V4-Pro-0813","source_effort":null,"source_model_version":"deepseek-v4-pro-0813_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"7bf486bf5712d82006953f542ace864b525550552fa55296648111523b31aeff","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":47.3,"score_display":"47.3","source_stated_rank":26},"run_fingerprint":"9ce4a9f88869c02588070491f6e4bd216dc3b4b847eb3aacef192ffb7e00636a","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7bbfeeac30a4f4702a8667b3","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"70358af9bd0ca1708e134f7c9eb1b11c58a078d8e3e0bf125fbf1d8fae7ccab7","metric_details":{"confidence_level":0.95,"license":"MIT","session_count":7090},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"DeepSeek V4 Pro (High) (0813)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8da2ef2a9965fcdd7d630c673d331c0ef821ae740f69ce4fe65705769f260fdd","result":{"confidence_high":0.028421064074707,"confidence_low":-0.0047129205344373995,"sample_count":797284,"score":0.0118540717701348,"score_display":"0.0119","source_stated_rank":27},"run_fingerprint":"554c77fdd56cef02dbe10dda4c73dab010969e451e663cc9b71cf4bd4c8461cc","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30663e63c085ce7e9e35d6eb","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a70a4dfb1ed22a4a9b237237f8f88e8bfe9137b523d7ac2a4623466ca26893eb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-13","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · gdpval"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"deepseek-v4-pro","upstream_model_label":"DeepSeek V4 Pro 0813 (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"504037b0de07d2e70d5bb80e5820a188ef9b5544f4426f8557a597c2bae94e7c","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1441.42,"score_display":"1441","source_stated_rank":null},"run_fingerprint":"23a0d4eaa5434d70bde2fdbbe8e4b95b2fa5d8c272821cfacc54f753b48dd8a1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_85d13172a494484884c0ddf3","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ec30c7999280e98e716acb7f79521af227767c66745f40af347f5eee705022db","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-13","reported_confidence_interval":"-18 / +18","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · gdpval"},"model":{"id":"deepseek/deepseek-v4-pro-20260813","name":"DeepSeek V4 Pro (2026-08-13)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"deepseek-v4-pro-non-reasoning","upstream_model_label":"DeepSeek V4 Pro 0813 (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"04cb82c43d3c16c0958a924e9934e26706d7ae036da9a61eef6cc66552815bcf","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1052.42,"score_display":"1052","source_stated_rank":null},"run_fingerprint":"b18613405ac7102528b54cee5ed072c73d2e8d52e972e3e523cf5b1dcf5795ae","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0813 (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_3152edaac5f1fd3d1ecc5bbe","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a3607d402720a766299b7c96fe6e0a1e952902df16b7f6d6bc02eb100f39667f","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-05"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Opus 4.6; model release: 2026-02-05","source_accessibility":"API access","source_model_name":"Claude Opus 4.6","source_model_release_date":"2026-02-05","source_model_versions":["claude-opus-4-6","claude-opus-4-6_120K","claude-opus-4-6_high","claude-opus-4-6_max","claude-opus-4-6_unknown"],"source_reasoning_efforts":["high","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e81ed1eec1ac4960a2b25fc3c3c48f3b97e729dc496c50a22ac04c28b5955b0a","result":{"confidence_high":157.25,"confidence_low":153.43,"sample_count":null,"score":155.31,"score_display":"155.31","source_stated_rank":27},"run_fingerprint":"e8a98e14a4fe14ef2e8a474903b8b0602e500214fe3807852fdf167431421483","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_152e3a035044e671415749f7","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bea940fff688eaaa40ac6f7d45278893201857b87d646dd4ffd9ef821504dae7","metric_details":{"license":"Proprietary","variance":3.0383237787276296},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1508.1455687105101,"confidence_low":1501.31282528494,"sample_count":77636,"score":1504.729196997725,"score_display":"1505","source_stated_rank":2},"run_fingerprint":"2fc8216eb7cdb502b7fe52f425169dd687cd16bc267fd2ae628ec68a77d5d760","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f3a928c2dc4d6711d402db03","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e0eb934d3454eb2869f976065a0e14b397d22bd8efaf2c8e2b372120366c2485","metric_details":{"license":"Proprietary","variance":2.877674099536261},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1500.5973658042683,"confidence_low":1493.9477142185845,"sample_count":82189,"score":1497.2725400114264,"score_display":"1497","source_stated_rank":7},"run_fingerprint":"f0c17f740aedcb954703e36e9c7d0c0c410f2dffd04ec9f687d478a52013f00c","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b1f6e6294ce20eeafb00b5f3","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"228fadab9cb4d4086c095ccac5f7e2d594b4bd23dae9662cc669b0b2d36eb954","metric_details":{"category_scores":{"Agentic Coding":48.989666666666665,"Coding":78.1845,"Data Analysis":69.893,"IF":63.3125,"Language":83.26966666666667,"Mathematics":89.31700000000001,"Reasoning":88.673}},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.51990476190477,"score_display":"74.52","source_stated_rank":41},"run_fingerprint":"d1877eb6997674aff47aa35b4db40a749dfe9368dde7bfc9f81ce3d0c0d42c79","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f3a928c2dc4d6711d402db03","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":229,"at":"2026-02-17T17:04:32Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17T17:04:32Z","status":"stale"},"measurement_id":"b84a725697920467ce70bcc84f39d56289d1ab678c2c67fbc07c03d7c0eec270","metric_details":{"calibration_error":46.0,"confidence_interval_half_width":1.86,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=claude-opus-4-6-thinking-max; createdAt=2026-02-17T17:04:32Z","source_row_created_at":"2026-02-17T17:04:32Z"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"claude-opus-4-6-thinking-max","source_reasoning_label":"max","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"2fefe34406515469499b0914b6d61437c27d3ea5c340e41bc9044f106279add2","result":{"confidence_high":36.3,"confidence_low":32.58,"sample_count":2500,"score":34.44,"score_display":"34.44","source_stated_rank":6},"run_fingerprint":"afd07823f631bbbec4a60941094da75df3459f5829ae29d9ceaa31993911a4be","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=claude-opus-4-6-thinking-max; createdAt=2026-02-17T17:04:32Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":229,"at":"2026-02-17T17:05:29Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17T17:05:29Z","status":"stale"},"measurement_id":"86c3b962904a91243d37c6e39716e71b0e6197b61e9a9507f8752a18750d4875","metric_details":{"calibration_error":44.0,"confidence_interval_half_width":1.54,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=claude-opus-4-6 (Non-Thinking); createdAt=2026-02-17T17:05:29Z","source_row_created_at":"2026-02-17T17:05:29Z"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"claude-opus-4-6 (Non-Thinking)","source_reasoning_label":"off","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a78d9a0696a65d2e910636cecf7bc9d357267fa564deaffc7063fe0638cbf951","result":{"confidence_high":20.54,"confidence_low":17.46,"sample_count":2500,"score":19.0,"score_display":"19.00","source_stated_rank":16},"run_fingerprint":"7643a6c0fee04691be429cb4eb1762b420b3243e13016b596a1e2882a47cb014","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=claude-opus-4-6 (Non-Thinking); createdAt=2026-02-17T17:05:29Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_b8468914be40cfa88e901fd7","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":"2026-02-05","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8b9b3afc25070c77e81a41d275853a922699fad3df2798a2c2f939af241e14bf","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/anthropic-claude-opus-4-6","source_model":"Opus 4.6"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3ca656b91b143309299fcaf0bf9ed00521021742d424de85b6c27066b736f206","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.2,"score_display":"34.2","source_stated_rank":20},"run_fingerprint":"74c9da86c9250ddd0a8f7fc4a51f63fdb87c6e6c00300adc2c0d27cd205ed7cc","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_97744021e528eec9843849c1","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:27:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:27:45.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5e236c95d85b69c11b8f45a70a62b22c423091950e223a6bde01560e2cea850b","metric_details":{"best_score_across_scorers":"0.47","model_release_date":"2026-02-05","stderr":1.5790799515836722},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"85fhMxLaebWtLoi6oLpTbi","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F85fhMxLaebWtLoi6oLpTbi.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/85fhMxLaebWtLoi6oLpTbi.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"c0cc28b2bc35c4d4af2d8b42dc97810da93d85493654fee2989049a865502419","result":{"confidence_high":50.094996705103995,"confidence_low":43.905003294896005,"sample_count":1000,"score":47.0,"score_display":"47.0","source_stated_rank":33},"run_fingerprint":"5cdd130ce5a859ba673ea555eb1728e722b021c8ecc0939b3e11f64270228dd6","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":209,"at":"2026-03-10T00:00:00Z","basis":"evaluation_at","evaluated_at":"2026-03-10T00:00:00Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"cf14bf0e6dabd8b390e74ea5d25e96142d1afd1f2359de68c261ae4d1a347814","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-opus-4-6-default","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8fffa40876fd686cc8648ad5da762f3e665a2d9d5d3a880a11c7c27237c9625f","result":{"confidence_high":48.30175495175897,"confidence_low":42.075703846752454,"sample_count":null,"score":45.18872939925571,"score_display":"45.2","source_stated_rank":13},"run_fingerprint":"4272f9d6dda9d14f8523f34e2d6072934674bd273c7d9c166b428a0f75adb0aa","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d6068788efe460192db330c7","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"89e4e3bcc2941cc846ee5ed0f862154158071a7c901dce02633b2e53fd14e5cc","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.294901,"date_is_proxy":true,"latency_seconds":319.02,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.186},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","few_shot_accuracy":88.384,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-6-thinking","zero_shot_accuracy":90.909},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"870e38a2db34cdd1ea3593dbf3d8a5b87eaa914778f769bb678b83f52441ce59","result":{"confidence_high":91.97056,"confidence_low":87.32144,"sample_count":396,"score":89.646,"score_display":"89.65","source_stated_rank":28},"run_fingerprint":"224daee447aab048df8893a93990cbfbecb60e024fec55b8752db00a56d58bc0","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T23:57:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:57:58.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"910e008eae9532af3b6bae615a357f3bd102dedabe5c3fd6f82e37f364bfa618","metric_details":{"best_score_across_scorers":"0.8838383838383839","model_release_date":"2026-02-05","stderr":2.282888177524936},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"JcQMgXdZKU54u52B8TGQKp","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJcQMgXdZKU54u52B8TGQKp.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/JcQMgXdZKU54u52B8TGQKp.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"61eafc68f1a15d5694331bb3080cdac9c076bcd8ec012acc89a67d6b897c382e","result":{"confidence_high":92.85829921178725,"confidence_low":83.9093775558895,"sample_count":null,"score":88.38383838383838,"score_display":"88.4","source_stated_rank":58},"run_fingerprint":"8f366d0bfcf7e94cfa48c370b3018ecc5e27b471409f7eb3844a7a2aa9d7da9d","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7fcc2f920e3e48e1cc45b9acfa39a11efa87b13f48de5fa194a153da905e8835","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.052129,"date_is_proxy":true,"latency_seconds":49.451,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.455},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-6-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"dda6e54cdd58083cf92e38799da4621d6c55c6d388276ea5b564f400f79a24b1","result":{"confidence_high":89.9988,"confidence_low":88.2152,"sample_count":null,"score":89.107,"score_display":"89.1","source_stated_rank":15},"run_fingerprint":"1d0519a0e294a046283297f7c7fa374020e172df6c44bb8ebafb0236d9181f82","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T23:58:17.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:58:17.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c8e5988bbc0ad10adb5ce7bd9c91a92e6f97c066472fe1a39ecea35bcfd2c335","metric_details":{"best_score_across_scorers":"0.9111111111111111","model_release_date":"2026-02-05","stderr":4.290254662948546},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"coxYk5vK5cBhQ7iZohMtA5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcoxYk5vK5cBhQ7iZohMtA5.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/coxYk5vK5cBhQ7iZohMtA5.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"91f1ee70583fe8a71c9a36a257a8ff7af9a54946c9765146dbd06d21cc083bd5","result":{"confidence_high":99.52001025049026,"confidence_low":82.70221197173197,"sample_count":45,"score":91.11111111111111,"score_display":"91.1","source_stated_rank":65},"run_fingerprint":"dcac183f8ca73baa60d3088d7eef5c531d2b4eeaf69ab90d30e9416926917622","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-11T00:19:44.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T00:19:44.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"924ff369ad55e0511384808e08724482cf4b487c5ec8f79cf75fca96daa8e6af","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.6596491228070176","leaderboard_private_problem_count":285,"model_release_date":"2026-02-05","public_example_count":10,"stderr":2.8116467881852296},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"kgJ2d6XbKj3dZutDcBYSKU","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkgJ2d6XbKj3dZutDcBYSKU.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/kgJ2d6XbKj3dZutDcBYSKU.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 4.6 (max)","thinking":null,"upstream_model_id":"claude-opus-4-6_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"4cfb756cf4e21c80a11a6c1a5fdd6d61e065e2c01ee1e7178cf2bc90817cd4fb","result":{"confidence_high":71.4757399855448,"confidence_low":60.4540845758587,"sample_count":285,"score":65.96491228070175,"score_display":"66.0","source_stated_rank":28},"run_fingerprint":"23a5c294436115d1e931426c9628f6960fbfe961d508766cf7ae504d7584bcad","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"35b4258187f95934b97143a66e0893b46535ee09ee4d6696af050ec2a7f232d9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":3.641,"model_release_date":"2026-02-05T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-opus-4-6-thinking","model_type":"CoT","upstream_model_id":"claude-opus-4-6-thinking-120K-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"064163ac017b920b561ddfc144ba380b6232580bcc756f1dc732224e4f3c6eec","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.75,"score_display":"69","source_stated_rank":57},"run_fingerprint":"cf87692b8838c99e9821395b69bbfdc2d4442f50a30775c10dc102a1e25d3fd3","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_3b724677f8ab5619a6947425","provider_config":{"source_label":"max","token_budget":120000},"provider_mode_key":"Max","raw_label":"max","reasoning_enabled":true,"strategy":"token_budget","token_budget":120000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d790bc4e326357a76f56b9d4442777e02d7ee76e100ced2c4d37602aa4ddb204","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":3.4719,"model_release_date":"2026-02-05T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-opus-4-6-thinking","model_type":"CoT","upstream_model_id":"claude-opus-4-6-thinking-120K-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f6ce6e8422c0135675beca552abc0e2d76fcf1ef4296d09638db8ddab4e7c8ad","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.17,"score_display":"69","source_stated_rank":55},"run_fingerprint":"748909b89f367ee9652ff8a4b8d74488ac024f6e28a115402f2d620d198d8311","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_2ced8384758eb0401656a3a7","provider_config":{"source_label":"high","token_budget":120000},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"token_budget","token_budget":120000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f39735195cd11b308265554dc44b5004da2792066379f3004f7ca76dddb9c1d6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":2.7209,"model_release_date":"2026-02-05T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-opus-4-6-thinking","model_type":"CoT","upstream_model_id":"claude-opus-4-6-thinking-120K-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"289353b7e5ac41c19124d714eb3a17b6ddf61548b2b63012674eecb7fc0c6d93","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.25,"score_display":"66","source_stated_rank":64},"run_fingerprint":"c6aff8fbd0e63e6b96029bee993924f7807876ecd5fede97dfdc41cff997e50a","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_aaa88a49d042219442747dc9","provider_config":{"source_label":"medium","token_budget":120000},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"token_budget","token_budget":120000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8afabe99bf1731acad861f9f13f2dbcc29515a410acef244da40b22fb5efbb66","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":2.2507,"model_release_date":"2026-02-05T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-opus-4-6-thinking","model_type":"CoT","upstream_model_id":"claude-opus-4-6-thinking-120K-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e9bb827b297332f7d64d79a9c93bd8c4faf39462f93c7505ddcf8d7854a55634","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.58,"score_display":"65","source_stated_rank":68},"run_fingerprint":"b53f16cddf5b049b0bcb94fe8b5bfa01bef8494fe570a47e0456423f41fecf84","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_21b27d1b402dcd758820491e","provider_config":{"source_label":"low","token_budget":120000},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"token_budget","token_budget":120000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-11T00:19:44.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T00:19:44.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1bca002b27ca8f13e443ad842d1f801306a41627c824be80013f17d2b7e256a8","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.2682926829268293","leaderboard_private_problem_count":41,"model_release_date":"2026-02-05","public_example_count":2,"stderr":7.005564203095157},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"2hbbp8z6PhB7c2T33os9q5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2hbbp8z6PhB7c2T33os9q5.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2hbbp8z6PhB7c2T33os9q5.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Claude Opus 4.6 (max)","thinking":null,"upstream_model_id":"claude-opus-4-6_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"977303192a8bf64f180a1f4395dc194dde5d277227ef9d8d4f9f292afbb127d7","result":{"confidence_high":40.56017413074944,"confidence_low":13.09836245461642,"sample_count":41,"score":26.82926829268293,"score_display":"26.8","source_stated_rank":37},"run_fingerprint":"3df2adab3738fcaf02faf09d21c44c4de01e9d64b39ee967d9e2b8798c3a3d4e","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ecda30b168a4b66d05cf0305b47f4abfc9689ff38ddc886d7e25a96f9caab80c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.116094,"date_is_proxy":true,"latency_seconds":119.005,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.884},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-6-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"98cf95f215a1fd786b4ef544fd89e51bc9ce11f68de2e992eb0ffc151431c5da","result":{"confidence_high":85.60564000000001,"confidence_low":82.14036,"sample_count":null,"score":83.873,"score_display":"83.9","source_stated_rank":29},"run_fingerprint":"5ce736ca45b25d451421cd80b60456fde25adf314c9952987805207c1bbe0276","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"718498d218e6d9d10dedc8c4e671a3d66c154cfd7e2696beb9efa5973744452e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"e4efa10f59195bbd55fde3aa51b8c9db51ed225321e6620142291c31b330a1de","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.6,"score_display":"67.6","source_stated_rank":16},"run_fingerprint":"b7f43cb7fad463e5a5fc30f333f690d90dc21923321ce2212101e40031daf4f6","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_774e289c6cfcaa16db364671","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T23:58:11.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:58:11.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"94cf6495b8444cf64098825aed99cf31cb08fb791c2c69afa74c22f7ec475970","metric_details":{"best_score_across_scorers":"0.14","model_release_date":"2026-02-05","stderr":3.4873508801977695},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FF8iU7vP5ecg6PdvDyE7XG","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFF8iU7vP5ecg6PdvDyE7XG.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/FF8iU7vP5ecg6PdvDyE7XG.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a67de8d600aaf15a8c2ab545ce3598554d2f65584867ecaa886faf8899188df6","result":{"confidence_high":20.83520772518763,"confidence_low":7.164792274812374,"sample_count":100,"score":14.000000000000002,"score_display":"14.0","source_stated_rank":113},"run_fingerprint":"71875107e4ececcd89d18c16529c1d0988d7e2a8b8fa26b7cf322b1129790650","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"048b84f3389e266e474f1d814287c6551073f187529cf5dda80da7642b163da1","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.216469,"date_is_proxy":true,"latency_seconds":350.76,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.848},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-6-thinking"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"c44b23a34180e2f551fdcfe004c17bcace814576880cb8595f1ea370d20db5ce","result":{"confidence_high":81.82208,"confidence_low":74.57792,"sample_count":500,"score":78.2,"score_display":"78.2","source_stated_rank":30},"run_fingerprint":"d3adc2ebb01e7ffd670e635a1d8aa46f020ded96e800028f837d67cb7eaa6879","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-18T20:07:57.705Z","basis":"evaluation_started_at","evaluated_at":"2026-02-18T20:07:57.705Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"426f0bc0d0ae28b7e77a30306f7675753bb682c27805a7402caaf95ceefd99ca","metric_details":{"best_score_across_scorers":"0.7871900826446281","model_release_date":"2026-02-05","stderr":1.862353730305288},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"2hH2GAPUFaJR7pfVKhhicg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2hH2GAPUFaJR7pfVKhhicg.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/2hH2GAPUFaJR7pfVKhhicg.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"331382efa1657302238488d569fd0d04ad0f52160ce85f97e342ebfc45a87559","result":{"confidence_high":82.36922157586118,"confidence_low":75.06879495306445,"sample_count":484,"score":78.71900826446281,"score_display":"78.7","source_stated_rank":4},"run_fingerprint":"4c452bc642fb907e54de3ef755c7d55a31837fffb959c34078037353c8bb246c","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_06f633219ece09d93fb1b7a6","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":230,"at":"2026-02-17","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17","status":"stale"},"measurement_id":"9193bafae08dab541ef654ac8b36a732c9c43954c8d9c4ff394fa3a96b185d23","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_claude-4-6-opus","logs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_claude-4-6-opus/logs","model_display":"Claude 4.6 Opus","model_org":"Anthropic","reasoning_effort":null,"resolved":75.6,"site":"https://www.anthropic.com/claude","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: claude-opus-4-6","Org: Anthropic","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_claude-4-6-opus/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"dc9e22eedb3fbfd36a4c4b6147cccd3749a72bb9faa42f07c24757addd83724d","result":{"confidence_high":79.160176,"confidence_low":71.649444,"sample_count":500,"score":75.6,"score_display":"75.6","source_stated_rank":9},"run_fingerprint":"eb49e12cbd4134c64baa67ab764f0374a61f82ac8f18a26693c67cf9bddcf9b5","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3ad5ad3eb071fa211a82ac40","provider_config":{"source_id":"official-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"a3257c378b89451b6ad59b3698f23e7943fd81ba6d7a4fb0ef6da9ac1e9a657d","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-02-05","pass_at_5":60.909090909090914,"potential_contamination":false,"sem":1.3666633071248102},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"Claude Opus 4.6-high__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"ee4e41d217d898acdf8d142ff31ef7c20f7cb11cad469979ca052b4bc43818e1","result":{"confidence_high":50.49684190014645,"confidence_low":45.13952173621719,"sample_count":null,"score":47.81818181818182,"score_display":"47.82","source_stated_rank":null},"run_fingerprint":"77bac46b049f106a32671c6dde0f7a1af34d593c009617f9a3b2a4ab9c5211b7","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f3a928c2dc4d6711d402db03","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-live","metric":"resolved_rate","name":"SWE-bench-Live · Lite","release_id":"release_9d2de9d5be7e9822af19f7b3","split_or_window":"lite-300-python","unit":"percent","version":"SWE-bench-Live Lite"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":49,"at":"2026-08-17","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:08:10Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-17","status":"fresh"},"measurement_id":"26c88646db6f0b815ebadb8375a3a07598cb60dc2c3a9b9472f9c41fb6562108","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 300 scored items with a 95 % Wilson interval.","leaderboard_script_url":"https://swe-bench-live.github.io/leaderboard.js","publication_status":"cleared-with-attribution","resolved_instances":75,"submission_url":"https://github.com/SWE-bench-Live/submission/tree/main/submissions/lite/sapient-slingshot-agent/v3.1.0/20260817-opus-4-6"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"Slingshot-v3.1.0","report_file":"reports-0605.jsonl","report_set":"lite","source_model_label":"Claude-4.6-Opus"},"run_count":null,"scaffold":"Slingshot-v3.1.0","tools_allowed":"agent-specific coding tools in the official SWE-bench-Live harness"},"protocol_fingerprint":"316ad8f92bb102c364f20b335f7b61c5fa87ed7e9712c42d2dc4373709d36dba","result":{"confidence_high":30.19526,"confidence_low":20.436911,"sample_count":300,"score":25.0,"score_display":"25.0","source_stated_rank":null},"run_fingerprint":"1da267dbc5703c9f3726ef5fca592bbd81cd0abcb8d9f42e41880ad6709252a7","source":{"content_hash":"e0226b6333c547885c15c6d53074d0ad5e2a7b99db2d6b54640d7d6537c242b4","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-09-30T18:53:18Z","homepage_url":"https://swe-bench-live.github.io/","id":"swe-bench-live","last_fetched_at":"2026-10-05T12:33:28Z","license":"Benchmark and harness MIT; aggregate leaderboard results redistributable with attribution (maintainers' confirmation 2026-09-02)","locator":null,"name":"SWE-bench-Live · Lite","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-bench-live.github.io/reports-0605.jsonl"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7258dfad95526e807989ecf7","provider_config":{"source_id":"swe-bench-live","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-live","metric":"resolved_rate","name":"SWE-bench-Live · Lite","release_id":"release_9d2de9d5be7e9822af19f7b3","split_or_window":"lite-300-python","unit":"percent","version":"SWE-bench-Live Lite"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":104,"at":"2026-06-23","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:59Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-23","status":"stale"},"measurement_id":"12b821a2355e0e2a3f9c5aae56b4b391fb0ff293cec23e0c9cd29d0888754be5","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 300 scored items with a 95 % Wilson interval.","leaderboard_script_url":"https://swe-bench-live.github.io/leaderboard.js","publication_status":"cleared-with-attribution","resolved_instances":189,"submission_url":"https://github.com/SWE-bench-Live/submission/tree/main/submissions/lite/ami-agent"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"AMI Agent","report_file":"reports-0605.jsonl","report_set":"lite","source_model_label":"Claude-4.6-Opus"},"run_count":null,"scaffold":"AMI Agent","tools_allowed":"agent-specific coding tools in the official SWE-bench-Live harness"},"protocol_fingerprint":"68d1fd99c01a8a818ace7d74bf25c80a0ec1ad439999966efe91d6364205fc27","result":{"confidence_high":68.26692,"confidence_low":57.404351,"sample_count":300,"score":63.0,"score_display":"63.0","source_stated_rank":null},"run_fingerprint":"c09d35a68bb29f0463fbb93e3329d3071f66ea5b7414cf0c8d21e281aa33597c","source":{"content_hash":"e0226b6333c547885c15c6d53074d0ad5e2a7b99db2d6b54640d7d6537c242b4","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-09-30T18:53:18Z","homepage_url":"https://swe-bench-live.github.io/","id":"swe-bench-live","last_fetched_at":"2026-10-05T12:33:28Z","license":"Benchmark and harness MIT; aggregate leaderboard results redistributable with attribution (maintainers' confirmation 2026-09-02)","locator":null,"name":"SWE-bench-Live · Lite","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-bench-live.github.io/reports-0605.jsonl"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7258dfad95526e807989ecf7","provider_config":{"source_id":"swe-bench-live","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":179,"at":"2026-04-08T17:04:50Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-08T17:04:50Z","status":"stale"},"measurement_id":"824b8f64043df30ed16e7efdf93162751f7ba465dbc2122255d4ce1405ab30cc","metric_details":{"confidence_interval_half_width":3.61,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=claude-opus-4-6 (thinking)*; createdAt=2026-04-08T17:04:50Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2026-04-08T17:04:50Z"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":true,"cost_cap":"uncapped","protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"claude-opus-4-6 (thinking)*","source_reasoning_label":"thinking-unspecified","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576","turn_limit":250},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"58754f772cb210581c78cc24ca18cb8d4e1679e6f7f3b35b45afdd3d6b6ced1c","result":{"confidence_high":55.51,"confidence_low":48.29,"sample_count":731,"score":51.9,"score_display":"51.9","source_stated_rank":3},"run_fingerprint":"6a2ad0bbe0e63b9b3a05389c7256b63667dca74ad8b742838c1da206ffa31a9f","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=claude-opus-4-6 (thinking)*; createdAt=2026-04-08T17:04:50Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7c38e5b05641500a1a65f2c8","provider_config":{"source_id":"scale-swe-bench-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_56ee3f7fe573bfcffb1c36f4","split_or_window":"vendor-public-own-harness-comparison","unit":"percent","version":"SWE-bench Pro · Opus 4.7 comparison row"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:16Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"0da5813f79ee98d1b17491e6107a9c914dfe8a7eb499d889f4c83cbf3a4a97c4","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"§8.2, SWE-bench Pro comparison"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic comparison row; original evaluation date and effort not restated","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"a7729a0e5eb61dc6818f553ae3c27ab774411cd5ab4ed7f414456d74a05c26d2","source_published_at":"2026-04-16","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Anthropic agentic harness","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"515c25c8753ade5ee1de5170d2c5f57ed3e637bb312cdc8ef1a4d3a51375a11c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.4,"score_display":"53.4","source_stated_rank":null},"run_fingerprint":"570e414cc08a053d6a25bb744041b8968647888802e2163fa94b18da3866eb2f","source":{"content_hash":"a3416021494c6882904f6a123b4a9d3761d9596b999d8356f9aaf644fcf7aa14","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf","id":"anthropic-opus-4-7-card","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"§8.2, SWE-bench Pro comparison","name":"Anthropic · Claude Opus 4.7 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/037f06850df7fbe871e206dad004c3db5fd50340/Claude%20Opus%204.7%20System%20Card.pdf"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ba6e3e6a11b686ebbbd71d32","provider_config":{"source_id":"anthropic-opus-4-7-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":4,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"secondary-re-report","verified_status":"secondary-re-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":161,"at":"2026-04-27","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-27","status":"stale"},"measurement_id":"ed296e5c1a8b9b40f61f8595181ee1eb4661e6a58b7c0c87270e9161712c244e","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"GSO Leaderboard"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6_high","source_row_date":"2026-04-27","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"1596cd4ede296ae563e39dff0e68f7793750895e0d1eac5471fb9e59bf002fab","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.199999999999996,"score_display":"41.2","source_stated_rank":4},"run_fingerprint":"101178c424eb89cdf4ea7ca3727455748998ff1639aed04641a4b324898ee895","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f3a928c2dc4d6711d402db03","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":161,"at":"2026-04-27","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-27","status":"stale"},"measurement_id":"0284cccf2ae95f58b3e53637017067b6f875ab568429db2b312638c3908d2ad4","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"GSO Leaderboard"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6","source_row_date":"2026-04-27","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"20b835e09f1792f93de8e7b29c384c5c2b881089aa82793a7a64e1f23dbdebf8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.33,"score_display":"33.3","source_stated_rank":7},"run_fingerprint":"d1f5d94d427caf1b9df0b6a0f95673ab43cea524d207e7c164f9099a0248a898","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49abb66ce68a2d86302dddbc","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":236,"at":"2026-02-11","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-11","status":"stale"},"measurement_id":"87b6e06bd253c4ca0b38785a43cc6dd58ff9baccd8bcb4682ef21f75b0de1204","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"GSO Leaderboard"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-11","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"eee551fe07bd3ba1979bf87fa153023b8ee652ca4349aaad3a6dbbe0d0928b3d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.33,"score_display":"33.3","source_stated_rank":7},"run_fingerprint":"6cbcd9f097e75d529be6cac42cc5d3fb288d4dd7db2cbd5cea7417c593507f9b","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_49abb66ce68a2d86302dddbc","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"70e5086bbf76f26450bcb1e545b9235e00aa75899dcd0084bf5da2bb7e16ed2c","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.6","source_effort":"high","source_model_version":"claude-opus-4-6_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"c748fa81cfe10c6144fa6f7d7562489be23172b94784012fabe2a240a05be6f7","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":26.640000000000004,"score_display":"26.6","source_stated_rank":30},"run_fingerprint":"9d6344fd9bd04267430ba14d1dffd31a8838ab48b43712e82a5a0fd3bc86dd8f","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f3a928c2dc4d6711d402db03","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b77f2cbb59a8c93b9f4d91664a31f44d2dd1f43aa7c2d990f55da94078e25103","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.34344,"date_is_proxy":true,"latency_seconds":324.486,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.02},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-6-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"1446bf97322d5e1a2bb897f6165814b912cd0df8978d00301475769b9ea98769","result":{"confidence_high":86.6752,"confidence_low":82.6768,"sample_count":null,"score":84.676,"score_display":"84.7","source_stated_rank":38},"run_fingerprint":"1c5050a5609eaec40f95703091f9674a103ce3413ca80e7d30c548de5d851568","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"930475404a8e1796e835ec4f16eefe5f51ec44a002fcb927d7e380b0bec70364","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.061688","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-4.6-opus-no-thinking","source_effort":null,"source_model_version":"claude-opus-4-6","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"bd3d060a508306ef4cc3f8ba4bfb9d72658f96c96a6845870b108e54d6fd01d5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":996.5,"score_display":"996.50","source_stated_rank":51},"run_fingerprint":"0a8a0617fcc54d59885f969f87c1084f7819ecdb1ceaeb21be157a4e6703910a","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_505fa327bb2ef52374adba4e","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"793ba54ee26df89a625e1961df8494c54aca20bdd0609196b5fdfc82667ddc25","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":8.687436,"date_is_proxy":true,"latency_seconds":1278.658,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.371},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":false,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-6"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"b1f1ca2453fc38bae6b1d2c037c35262092dc75c4ff2bd8487bdc180099602ab","result":{"confidence_high":66.14016000000001,"confidence_low":49.00584,"sample_count":null,"score":57.573,"score_display":"57.6","source_stated_rank":46},"run_fingerprint":"3be4fcc11a52a6afbfe54baede030dca3d2f330f5adbcfa71e639e943f5bcf83","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"37922424655744504078f6d837d0b7f28ec275993c82c35207c1cfdcb4896792","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":8.279062,"date_is_proxy":true,"latency_seconds":1386.056,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.678},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":true,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-6-thinking"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"9197e7c2a6118d9badd2281eb4eaf43936ed196924725520b390b56000d424a2","result":{"confidence_high":62.66688,"confidence_low":44.329119999999996,"sample_count":null,"score":53.498,"score_display":"53.5","source_stated_rank":49},"run_fingerprint":"8473b9cda249e3207f63a38fce8686d4a36851cc68087a8c73083d5b1e34679b","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"598f02e74330e16cb027841da71baada10b01bf1ef654ee16e9c5d6a327623f5","metric_details":{"license":"Proprietary","variance":7.726245992183965},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-4-6-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"afdd70be6b14a468677b54e9688916f9b126860055c1d84c621f78f5d3717630","result":{"confidence_high":1551.3619783568224,"confidence_low":1540.4660973336968,"sample_count":19519,"score":1545.9140378452596,"score_display":"1546","source_stated_rank":38},"run_fingerprint":"e0fd9136d00880b4de82abd1403a09f0e0549ed18f14f1ab4c9dfb902650f227","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f3a928c2dc4d6711d402db03","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a2ff002c7417c152b6ec4e99cf9aa44ef87a28a478c4556927418a4a59dac8a9","metric_details":{"license":"Proprietary","variance":7.4118425786285735},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-4-6","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cbf0c31ea9541576ddcda5bf5f7a69dc44b7cd05983f6e6bf7da047d1d861a7b","result":{"confidence_high":1542.3642259821972,"confidence_low":1531.6923399162258,"sample_count":20762,"score":1537.0282829492116,"score_display":"1537","source_stated_rank":41},"run_fingerprint":"fcb1a32663f859b80919aad06b8a52a2005abdce110f97b009b020d50c39aef4","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_509bfdd7caf40d268e2320db","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"096b7e90eae28a013c28ec45881d2e64c77a30b3945eb7d904e129703380b43d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"8ff0bdd9a0c10f5fc5f083b148e751214d0a177e754ce40c030124de84c6a3b8","result":{"confidence_high":1552.21,"confidence_low":1541.03,"sample_count":18332,"score":1546.62,"score_display":"1546.62","source_stated_rank":35},"run_fingerprint":"471f3df060fb7077661ac65e4907cf8735f3ae167302cf9ee4416db05f6ab54a","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f3a928c2dc4d6711d402db03","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f2f9b91b1976a728a2d68e76ef43f115c03def975df67e7e4be4508f8d3cef4d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"630732a819384b1e94e57bf0f12df76bd42070cdadc3c516401c201e8ee5f99d","result":{"confidence_high":1542.59,"confidence_low":1531.67,"sample_count":19536,"score":1537.13,"score_display":"1537.13","source_stated_rank":38},"run_fingerprint":"5a808b3ecf5ca7479be81c15e3942432f0944d8df7c2f6817a01d450b85b1d92","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3dce7af9022832c0c4ca74e1","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"acb32df8a7d2c1bd1b6e9e2086ed9dfd09d7280d997d73298b95531bc4181a06","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"df4f0bee4d8e328ea708c8aa3aea2150c450af47c8fb173c795a5c7d77228787","result":{"confidence_high":1570.13,"confidence_low":1542.44,"sample_count":2553,"score":1556.29,"score_display":"1556.29","source_stated_rank":31},"run_fingerprint":"4431fe9f4038fc3e20ca7fb89c181c48827d9dc4324200ecc1654f7be44dcb48","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3dce7af9022832c0c4ca74e1","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"821b21d4ff507bdf3cce63133eb5f749ad859c56608208476415c924fbef6907","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"e220d5173b7f9c3f85bf1d864cf45f1646f627d8293a8c535d4b9696d26b25d4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.95,"score_display":"78.0","source_stated_rank":20},"run_fingerprint":"7144f8ff425161a126db961185bdb8a03d59c29accba390d9422d93dad778d86","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f3a928c2dc4d6711d402db03","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1499c055b9eefe9fb4f1ce73e08f62af032dd89c0f405cee166388f7976f1a17","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":"Adaptive thinking","upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"0cf958fcf86512efcc2f651bcfd31e49c497c4f1340d99f0a8a06df6cc44f24a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.9,"score_display":"77.9","source_stated_rank":21},"run_fingerprint":"1c233fc8b25aaad89255b78bbf67cfcc46b54535823310c1efc2958c1a82e443","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_55906fd12d487c556a8e238e","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8bdc188bf15853d4d1b9d1dd8e65f013343c937e8e3f79e9cc2b42326a97b9d3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-6","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"47894c80ccdaf40dfd0eaabb9dd237d477769a6977e1ed7cd32f006f7876eb11","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.87,"score_display":"65.9","source_stated_rank":41},"run_fingerprint":"acd4524f76e603fa6589637b58bde8f3890745d8d4fdb9bff78ad06efdca1688","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_55906fd12d487c556a8e238e","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":207,"at":"2026-03-12","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-12","status":"stale"},"measurement_id":"09f3da86d484d1cc183001f9d9114ae19aadd5f819d3f69f0f3738bb28724125","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":1.6,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-03-12","source_scaffold":"ForgeCode","upstream_leaderboard_url":null},"run_count":null,"scaffold":"ForgeCode","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"48bff76199f218c29efd01b499e12e5f448a3363bcfa8ba9ca39bca25d332249","result":{"confidence_high":82.936,"confidence_low":76.66400000000002,"sample_count":89,"score":79.80000000000001,"score_display":"79.8","source_stated_rank":8},"run_fingerprint":"87b41202c0b240b305d84ab96cd13c5c9b81dbdc78da1a4c2a11a777ced1efef","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"72b324cf18a91fe6c8849e5df5d22192ba09af551008dee69e7d87fe6d6b499f","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.4324615274,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"Meta-Harness","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Meta-Harness","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"d41831507344aadc2ba371b4ee5c5660dad352609d2649d2c7373902edca10a8","result":{"confidence_high":81.172118975704,"confidence_low":71.636869788296,"sample_count":89,"score":76.404494382,"score_display":"76.4","source_stated_rank":13},"run_fingerprint":"1c526dd328a6af887a8a101d2a2323ecd792bd57eb7f400f8ff953e73846c944","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"66bb6ffc3045b7e2da7ebe63d00bcec50b668b6123dd718f5ffdeb4b014efada","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.3922527823,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"Capy","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Capy","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"c6b6ea8136d8c7aea1a9cf1a8469975eb45c5246f5a293618ea0996e11d933f3","result":{"confidence_high":79.969714329708,"confidence_low":70.592083423092,"sample_count":89,"score":75.2808988764,"score_display":"75.3","source_stated_rank":16},"run_fingerprint":"0e6a320574b754f45750cd7c1cd835e0c444c7f17d7c2ce1371b31f28f66a7ef","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"9bbfd4cb488936064f529b416a7c6ead1e2b6d690f41c496ccd9593439534c64","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.5964111461,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"Terminus-KIRA","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus-KIRA","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"4b2f0e65215a82283e90065121e23113fd1a0fb347ec9fa1e03bbb11c78ef066","result":{"confidence_high":79.808066969956,"confidence_low":69.630135277244,"sample_count":89,"score":74.7191011236,"score_display":"74.7","source_stated_rank":19},"run_fingerprint":"2b577fa628f20ed0ef64e8fc54d6d8cb81d63c525dde280650d60568cf4d4157","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"d5214f1b333de1e7c657155706d8361c78c5576e72a8c06109f01f969c02c28b","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.208661072,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude 4.6 Opus","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"MAYA-V2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"MAYA-V2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"48ed769ccbfcbc2c75cab7455ca6a6419e6b8acd05cd289ad3af05adf12c4922","result":{"confidence_high":76.40762738652,"confidence_low":67.74967598428,"sample_count":89,"score":72.0786516854,"score_display":"72.1","source_stated_rank":21},"run_fingerprint":"c0b545c60766d23071f8d6c746dd8333fa27d51fc84fb805291995b25c5331e7","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"d20d8df74f9254feb807826202298d370d1ddb8164fd6aa62e47ffca0faf828d","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.6791493392,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"TongAgents","upstream_leaderboard_url":null},"run_count":null,"scaffold":"TongAgents","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"d51b0908caa7fa607db73b32e52588872add1d5720fbfe25ac03d7d4dd819b7f","result":{"confidence_high":77.16124506443201,"confidence_low":66.658979654768,"sample_count":89,"score":71.9101123596,"score_display":"71.9","source_stated_rank":22},"run_fingerprint":"d05e47e586bdfa6c1784b537e64cd5cce10c739a521ad240fb4afcce50787d57","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"d3f85654cb50c38ec5e40f71202b313f918e2ee40ff1dbb0307cf148712cbced","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.5,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-opus-4-6","source_row_date":"2026-02-05","source_scaffold":"Droid","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Droid","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"56499cc84e9ad8a276e1dc785bb49e2c9ff466d42f242f8fc0088c9eaa8ecd90","result":{"confidence_high":74.8,"confidence_low":64.99999999999999,"sample_count":89,"score":69.89999999999999,"score_display":"69.9","source_stated_rank":25},"run_fingerprint":"ddd19637cfeb85241df4093d30554086fb517fd1af7ab0845ad4f3d09ae4c7e6","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"f6b41bd52caca0724e456944ddd5f24c0f644fb2d0a56986c9b14d889f8f8dfb","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.4720163409,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"Droid","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Droid","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"5d97509ea6fb122dd11ba2409edd7ad5b1095a313649714c01e91125ad5963ed","result":{"confidence_high":74.73279247756399,"confidence_low":65.042488421236,"sample_count":89,"score":69.8876404494,"score_display":"69.9","source_stated_rank":26},"run_fingerprint":"741a23153b50338c8862ae42a8f38b435f6acbf0c205f2e0248bb05828c750e6","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"100f96119ea590713521e4900f22ed82fde717a9ccb5b397294a59ffc4af1bae","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"Crux","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Crux","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"1d61cddfed71e571ce06dba3a7cefc38ca0aed4fb47c03cd26c6bbbd7d7460ae","result":{"confidence_high":null,"confidence_low":null,"sample_count":89,"score":66.872659176,"score_display":"66.9","source_stated_rank":29},"run_fingerprint":"97f855ae3e30c93a333ec5aa3e06d5615c834470cec7c76ad4d9a794ee1118ad","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"02b83470a7dba605e4ec8d750be65914df6f6992013180b1776e3bd238574a03","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.5301893904,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"Mux","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Mux","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"7d450caf0936eee5e463a970fec37749e4ef17e2e22403f82ca9aacd77675ff0","result":{"confidence_high":71.476025137784,"confidence_low":61.557682727416,"sample_count":89,"score":66.5168539326,"score_display":"66.5","source_stated_rank":30},"run_fingerprint":"9721bca15a05a8f083b618db751c365e930d7307145808f09272dda0fc12ad76","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"8bb8eeb4c479345aa151bdbbc7c7d9efb82af3e6b36155b4675f80852ccab9fd","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.6609854054,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":11,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"55afb7f57cc7f60ee6da90e33df9c2753860db9fde1ebae8432a9a643c8bf947","result":{"confidence_high":68.136879709184,"confidence_low":57.705816920016005,"sample_count":89,"score":62.9213483146,"score_display":"62.9","source_stated_rank":40},"run_fingerprint":"19189c82c28296832ff89d8382a9ea66d9e0fe72687ef0351981c2c6bbd59a06","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"40f9cd1559addec28346d7a1fbca3f3c3908e914a293be0279eaa7b03e547555","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"standard_error_points":2.9049635593,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":12,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":"2026-02-05","source_scaffold":"Claude Code","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Claude Code","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"3154bd58e262d28bca8e02969c0a6beb377188416d625e306c68513c93010255","result":{"confidence_high":63.671256666127995,"confidence_low":52.283799513672,"sample_count":89,"score":57.9775280899,"score_display":"58.0","source_stated_rank":54},"run_fingerprint":"0c13cef4a853b37b0a708d410c1af36f257c142227778e10bfa33a1e47132d75","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00d545243940ce8ce2d1d4a6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":152,"at":"2026-05-06","basis":"evaluation_at","evaluated_at":"2026-05-06","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a367fed97f1422c645033d1fd6a59f0a9f2c1e609116743d1e496584c7b1b029","metric_details":{"cost":null,"pass_2":18.73,"pass_3":14.18,"pass_4":11.34},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"claude-opus-4-6_sierra_2026-05-05","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"5a2403363a0c9feff2c6ac7d53571698b24966266565d4704bcda085179eb000","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.32,"score_display":"27.3","source_stated_rank":null},"run_fingerprint":"5ba7ff191e3a0a55ee60408e5e13882a4bb8e18e8fd017270f4cbb447200b7d3","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":290,"at":"2025-12-18T18:01:17Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-18T18:01:17Z","status":"stale"},"measurement_id":"fe242d1a96b1599291937f4c7c83d6e71eb9af37cb582616e271887856c54098","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.7,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=claude-opus-4-6 (max); createdAt=2025-12-18T18:01:17Z","source_row_created_at":"2025-12-18T18:01:17Z","task_pass_coverage_threshold_percent":75},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"claude-opus-4-6 (max)","source_reasoning_label":"max","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"2df36ed98b93c9debcd38e15c8909c4601d7bb9662688feaf66fd550fcf7aff8","result":{"confidence_high":79.5,"confidence_low":74.1,"sample_count":1000,"score":76.8,"score_display":"76.8","source_stated_rank":3},"run_fingerprint":"213f8362527bd190572b59e6754ca44120bafcc9b30346a690b85906bc3671ba","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=claude-opus-4-6 (max); createdAt=2025-12-18T18:01:17Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"88a9abd13a51c873609fbf421e67c4c718628c1ec17b6a56c3ab18ee27530516","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.618","mean_standard_error":"4.8","model_release_date":"2026-02-05","notes":null,"standard_error_points":550.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"ebf2d00363a702f80999be21286fbeb3334ab283b90d40c78749e89280548d21","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":46.3,"score_display":"46.3","source_stated_rank":28},"run_fingerprint":"369638779c159075c024cce049a2339f6116316865a1ad8ce4cea3bc19be4399","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_850fc6175572e341e7de5b0a","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"fb75d625a137a11fdaa896441b4cc618cbbbb9f1db42e89870f37bd18664c7a2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.48700000000000004","mean_standard_error":"3.5","model_release_date":"2026-02-05","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.7,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.6 (High)","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"15f1075ac8e83eda39782e705302b642e585e2dc0f6189db699cd5a78811b6cf","result":{"confidence_high":39.652,"confidence_low":25.148,"sample_count":480,"score":32.4,"score_display":"32.4","source_stated_rank":24},"run_fingerprint":"eb8a599fc51d3c6b5441b0aa22944632970d169305dc6f618756c89e9f3ec702","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f3a928c2dc4d6711d402db03","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"appworld","metric":"task_goal_completion","name":"AppWorld","release_id":"release_3fa145932216c0afcdd9dae6","split_or_window":"test-challenge-all","unit":"percent","version":"official-c1f56015cf7c"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":196,"at":"2026-03-23","basis":"evaluation_at","evaluated_at":"2026-03-23","first_observed_at":"2026-09-02T21:28:04Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1cc1db4199d03d47da0120c97823e9d463f446b46557bf3b7f7ff8ed2929fac5","metric_details":{"interactions":20.52,"normal_task_goal_completion":91.1,"scenario_goal_completion":64.8},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"leaderboard_row_id":"f072cd07-6ae0fbdf","method":{"name":"Button Agent","tooltip":"Button Agent"},"repository_revision":"c1f56015cf7c3441ff1933f5bfbec879798d7bbe","source_model_label":{"name":"Claude Opus 4.6","tooltip":"claude-opus-4-6"},"submission_url":"https://distyl.ai/"},"run_count":null,"scaffold":"Button Agent","tools_allowed":"AppWorld APIs"},"protocol_fingerprint":"72b8a94a56270e5c3983fc0a16989987c068c2c329242dd70533a415bc929b59","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.4,"score_display":"72.4","source_stated_rank":null},"run_fingerprint":"6c665940b7aa160398a5b32af2a17e08b0123d7e0dbddc617029489a17385a15","source":{"content_hash":"d44080bb01254150fab30f845d22262a28ee7c90f7850b98ce0ab235ea043f0f","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-02T21:28:04Z","homepage_url":"https://appworld.dev/leaderboard","id":"appworld","last_fetched_at":"2026-10-05T12:33:29Z","license":"Aggregate server-evaluated scores redistributable with attribution (2026-09-02)","locator":null,"name":"AppWorld official leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/StonyBrookNLP/appworld-leaderboard/c1f56015cf7c3441ff1933f5bfbec879798d7bbe/experiments/outputs/_leaderboard.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e22277a9c302417e104ce78e","provider_config":{"source_id":"appworld","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1e3f7dfb5d6f02ffc67e6835eb098188cf46a346df345d115716b063bd14fdbb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-05","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-6","name":"Claude Opus 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.6","source_effort":null,"source_model_version":"claude-opus-4-6_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"5984834f42e164be2902d2931446c87ab2d68e88835f3ba18c49a083fba222b2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":8017.594,"score_display":"8017.59","source_stated_rank":10},"run_fingerprint":"1dd8e322feebff6a8e73414e70bae60a53e0d6ee9b4d3d3f8ddfc4f5174acea3","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1f201fa792617714eed8a8a3","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"067499b3faf88324c9fbc4094cca4132c0397e67cfe477ec1761549921250744","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-09-01"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":false,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen3.8 Max (0902); model release: 2026-09-01","source_accessibility":"API access","source_model_name":"Qwen3.8 Max (0902)","source_model_release_date":"2026-09-01","source_model_versions":["qwen3.8-max-0902_xhigh"],"source_reasoning_efforts":["xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0ac144278e3e64900134e6ae64ca2cbac16131bfb9928ad852a3f2079449fb33","result":{"confidence_high":157.33,"confidence_low":153.24,"sample_count":null,"score":155.17,"score_display":"155.17","source_stated_rank":28},"run_fingerprint":"50797b2b184b851d0bd82e850d0014c31195d91f2308ff5ae03ae97c7cacdb3b","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_164cbc78e507602d5ae2f029","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ade0a191f3ac52b498bc1da5971a08e52fcc82c2e8c341d77e0a6202e076d024","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":983616,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Qwen3.8 Max (0902) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max (0902)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"a5880235cc0617bf24858d518b05c3f0fec2473cd946245116e26bc9b8cfa04b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.415208,"score_display":"45.4","source_stated_rank":null},"run_fingerprint":"4efa33ddbbb33e318dd8018f4f856754e7691fa56a56396bcc68e96900c71f20","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max (0902) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fb817aa80071f23551b9a00e","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"77c70212f51433253868c50065742d8cc67f502f3f6b44ec31599269597eada3","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":983616,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Qwen3.8 Max (0902) · hle"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max (0902)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b7fe0606535e26e3011efb0871663621c2dc6b3c6fb7b93ae0365e6035bcc196","result":{"confidence_high":45.195297,"confidence_low":41.020159,"sample_count":2158,"score":43.095459,"score_display":"43.1","source_stated_rank":null},"run_fingerprint":"2bf7c806002d0600a80fd711934e0a1c620d40b0191666ce812424e567bb7c0a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max (0902) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fb817aa80071f23551b9a00e","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T18:04:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T18:04:58.000Z","first_observed_at":"2026-09-09T22:25:13Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b27ea5a3c91a1fa14726d7f52d5d31942a2a55ffa0e95ff3775301c63d669ab4","metric_details":{"best_score_across_scorers":"0.473","model_release_date":"2026-09-01","stderr":1.579621855130273},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"nBcLJJomVoeg2zh5F8Mnsu","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"7cad23dc5e0345efa51a17c61f2b76c4524b68b9bf3f9a669774a1e64a094e38","result":{"confidence_high":50.39605883605533,"confidence_low":44.20394116394466,"sample_count":1000,"score":47.3,"score_display":"47.3","source_stated_rank":32},"run_fingerprint":"393c20b0a45d172c85dfde5ee5c25112f5b59abaf9401277ae7d81d909163d37","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_164cbc78e507602d5ae2f029","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e315c41d2bc89f90223ae4ab70b24f0177649c2c04b20bee10b121b5c64590ae","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":983616,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Qwen3.8 Max (0902) · gpqa"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max (0902)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"e15ce0a38df4380147240e234dfb3d3815b79c928f2a371f0f645021e2368478","result":{"confidence_high":95.664893,"confidence_low":88.361393,"sample_count":198,"score":92.828283,"score_display":"92.8","source_stated_rank":null},"run_fingerprint":"57f12c5729b19375fee335b5502319ecce76d61df56860b82e0d111a993822ea","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max (0902) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fb817aa80071f23551b9a00e","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":32,"at":"2026-09-02T18:04:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T18:04:58.000Z","first_observed_at":"2026-09-09T22:25:13Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b8bed638fe4e3fe33df49e8a848481d8606096903d3e155d270f23ae10df37e5","metric_details":{"best_score_across_scorers":"0.922979797979798","model_release_date":"2026-09-01","stderr":1.7249976996605707},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"VzeQXKBH9maR2nDcZJ8b7S","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"0f6c7f5b7f23c227864b58e1f8af4e6cd587fb05433793a289d8e744197239d9","result":{"confidence_high":95.67897528931452,"confidence_low":88.91698430664509,"sample_count":null,"score":92.2979797979798,"score_display":"92.3","source_stated_rank":26},"run_fingerprint":"08216c0fb9eabbc38d0536a86e63b540533380eaeedfc9b5bf1c54f1265f3279","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_164cbc78e507602d5ae2f029","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T18:04:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T18:04:58.000Z","first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"42d7ed4b7a07152bd9542ffb0275ffc69708d174688314b2efbb7f160e8c0129","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-09-01","stderr":0.0},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"2f8Vn5QBdJnc9i5y9jzXS7","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"540abe7eb1643f4b49284343fed302469c7b111c03a249b3fb97f0619b2a8dc1","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"4d220b3b9e864792c5a3ee4dd69802b4a006c0155171b5a1678ae5b01b1453a1","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_164cbc78e507602d5ae2f029","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T18:04:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T18:04:58.000Z","first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d3f16521873d9e9c107182225d0c6dc445ed59d8a0a000a7aa27ffe86adf7f74","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.656140350877193","leaderboard_private_problem_count":285,"model_release_date":"2026-09-01","public_example_count":10,"stderr":2.818576398907258},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"FpQf5xBwJPb4ZEo6SgdKWq","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.8 Max (0902) (xhigh)","thinking":null,"upstream_model_id":"qwen3.8-max-0902_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"15eb55a29c8b986596dfb10383777044168822a0790707a85eea498f76ca2d71","result":{"confidence_high":71.13844482957752,"confidence_low":60.08962534586107,"sample_count":285,"score":65.6140350877193,"score_display":"65.6","source_stated_rank":30},"run_fingerprint":"c683346f65607cfd736814bec2f24567447ed45eea588b43aa885431bea9b3f7","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_164cbc78e507602d5ae2f029","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T18:04:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T18:04:58.000Z","first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7e83cbb47fddc930192b0cb9a76c3cafa9bb90332eaad75302a568fca3d9b06d","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.34146341463414637","leaderboard_private_problem_count":41,"model_release_date":"2026-09-01","public_example_count":2,"stderr":7.497768853141171},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"kZUdrqdGtCEJa3ip6L9YnH","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.8 Max (0902) (xhigh)","thinking":null,"upstream_model_id":"qwen3.8-max-0902_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"3929c6c9d56ad502dc10a72e084c4ff9e06c385d940b2b5bfe9df5f695a338e0","result":{"confidence_high":48.84196841557133,"confidence_low":19.450714511257942,"sample_count":41,"score":34.146341463414636,"score_display":"34.1","source_stated_rank":29},"run_fingerprint":"61b694c6eead5bacc25057b41f268b1819f458d45434278ea37d0ad982080330","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_164cbc78e507602d5ae2f029","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eabdd48c8c7a1c18b22fc36fd94090a9ba2e5d585421347e7b6e946923291a46","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":983616,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Qwen3.8 Max (0902) · mmmuPro"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max (0902)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9400b85e4d104cd85cdf1a018f3bcad26b73fe6f7b631a48ef0754ab722f2e82","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.774566,"score_display":"82.8","source_stated_rank":null},"run_fingerprint":"ead0a07385c5d00117bd26eef3a0e095fe80de7c223e26f5e2b40c1f6064c1cf","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max (0902) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fb817aa80071f23551b9a00e","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c6edc7d2f5d5f8c6f38babdf4db5101dabd7caf02da3c8d55e7cf6e7ea4cea04","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 Max (0902)","source_effort":null,"source_model_version":"qwen3.8-max-0902_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a0ebb77a59f59ccaa16ca5ae119f6bd278c6d21b5a4d079aef7548607a6fa80f","result":{"confidence_high":28.178675,"confidence_low":10.564331,"sample_count":71,"score":17.714285714285698,"score_display":"17.7","source_stated_rank":61},"run_fingerprint":"ce9bbef1bb1f2b3cc2ec5a731fd1c121f8147826a6bd6b40250db929e565ee0d","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f1de9baf1c06fe65d3662dc1","provider_config":{"source_label":"max"},"provider_mode_key":"Max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dbba95e9dbec0056ba89addb3db8e6c2dcfeb26e0b24e0cdbb0e78e5e777f244","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":983616,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Qwen3.8 Max (0902) · critpt"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max (0902)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"cb112066f682cd08311548559454a1823fcf10f99d740265fef0c19a53527f9a","result":{"confidence_high":28.262679,"confidence_low":10.525214,"sample_count":70,"score":17.714286,"score_display":"17.7","source_stated_rank":null},"run_fingerprint":"b0937934f04d5f5a7a8d7ad622c2d9206b180a781016544ba2d149bcec6db7c8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max (0902) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fb817aa80071f23551b9a00e","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T18:04:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-02T18:04:58.000Z","first_observed_at":"2026-09-09T22:25:09Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"32a32a8fa4603dc93832950dc2f1f0afeccfa5eef72f130392d9b26cf371418f","metric_details":{"best_score_across_scorers":"0.4","model_release_date":"2026-09-01","stderr":4.92365963917331},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"8qGBfpdQXMmLcSqPwz7ofm","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ba5dd0992dd67bc06004813c8fbf2ad5a63ed8d9e5b79f4b5cd037146774e6da","result":{"confidence_high":49.650372892779686,"confidence_low":30.349627107220314,"sample_count":100,"score":40.0,"score_display":"40.0","source_stated_rank":24},"run_fingerprint":"208849005eb3fd1050d0f4b05ca64aa7f5444621eb8536bdb151306a132b0024","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_164cbc78e507602d5ae2f029","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3050cb200a675104c8ff5317e379ea9e5d6abbd03cb1147b64a09e68784b953c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-01","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 Max (0902)","source_effort":null,"source_model_version":"qwen3.8-max-0902_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"9d299d21e0ff661f498a2b851f71bc27ba9719c7c26e255b52bfa2384e3ee0cf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.0833333333333,"score_display":"52.1","source_stated_rank":72},"run_fingerprint":"15e41a3b4d9f02a5a909332dbbcdca7e11c6b177e39627c991a9430a81258420","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f1de9baf1c06fe65d3662dc1","provider_config":{"source_label":"max"},"provider_mode_key":"Max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"12f4db2e56fd406a2c5508f787553691b6b50b29492f0d1d1535ce8d41e16082","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":983616,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Qwen3.8 Max (0902) · scicode"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max (0902)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"78b518a7f9abcbd72b3884744cf80c58c7392989ce7a4d0e054f4ca3ab4ebf2c","result":{"confidence_high":57.787566,"confidence_low":46.324253,"sample_count":288,"score":52.083333,"score_display":"52.1","source_stated_rank":null},"run_fingerprint":"488279ce6dd03c4217d27f3dc510f8e16ec88f95dfc982b95a558dfd7140ad7d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max (0902) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fb817aa80071f23551b9a00e","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5e855da9bea8094c8bc37ebf56cf029d98932e0a28d76532e238ac846a9c7a72","metric_details":{"license":"Proprietary","variance":15.898763022821441},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.8-max-0902","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e02afc39563e20218e25092403966c525d5b7e1bb84db957315c1bc5c3152b47","result":{"confidence_high":1677.826703618761,"confidence_low":1662.1966756658567,"sample_count":9485,"score":1670.011689642309,"score_display":"1670","source_stated_rank":11},"run_fingerprint":"d68f484b03500528f5beba8b22e7609dd6140e1aa4759e3782ce822d4ece9dc8","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_30a83a958902d8055b81e819","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cf85810ea59bc035f78404af2f5b4619a0b0bbb4ccfceecfce8ae6d98fd35b45","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":983616,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","source_locator":"Leaderboard models table · Qwen3.8 Max (0902) · tauBanking"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max (0902)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"07b678c5d1e2cc9ddb42c313474343945d7eb17f08989b8dfe6a8c133c3d5973","result":{"confidence_high":57.667743,"confidence_low":38.16731,"sample_count":97,"score":47.835052,"score_display":"47.8","source_stated_rank":null},"run_fingerprint":"ebb1a900b5de1434a481ee3cbbf29b3d656e188f1206c27de32a56c55e10b108","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max (0902) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fb817aa80071f23551b9a00e","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"76c09952dda8406140523fe767a3c54370f918f8c255771ca98e921d3b08351f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":983616,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-02","reported_confidence_interval":"-21 / +21","source_locator":"Leaderboard models table · Qwen3.8 Max (0902) · gdpval"},"model":{"id":"alibaba/qwen3-8-max-0902","name":"Qwen 3.8 Max (0902)","provider":"Alibaba","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-8-max","upstream_model_label":"Qwen3.8 Max (0902)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c8d6cb183e33277e2126718d54cc4b135d9bb3d1aee8912aefb0dc4633060f29","result":{"confidence_high":1692.79,"confidence_low":1650.01,"sample_count":220,"score":1671.4,"score_display":"1671","source_stated_rank":null},"run_fingerprint":"a261b2b17f973695ab4bab223ba759fc4d047d51ef01513185ad4ef59a3b0844","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 Max (0902) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fb817aa80071f23551b9a00e","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a07842541f4d7c9310e8d3c8944675f85d5b6ebba70a9ae81d21e296957d57e4","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-09-09"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: DeepSeek V4.1 Flash; model release: 2026-09-09","source_accessibility":"Open weights (unrestricted)","source_model_name":"DeepSeek V4.1 Flash","source_model_release_date":"2026-09-09","source_model_versions":["deepseek-v4.1-flash_high","deepseek-v4.1-flash_unknown"],"source_reasoning_efforts":["high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"114a8673cd8eb3d1c1f644aff51275a9b95dc8e51452393f5923611846f962da","result":{"confidence_high":157.56,"confidence_low":148.77,"sample_count":null,"score":155.0,"score_display":"155.00","source_stated_rank":29},"run_fingerprint":"ac53536a5c8b2e3beaa39fae298453ae758ad02df76dde4592f2a70f26ed7d10","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dd74f317b70c8284099eda37","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"68baf94279af5375b9dd5f24dadbe039ac882a9c4cfb55fd7f08ed5cbbd56b4c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · intelligenceIndex"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash","upstream_model_label":"DeepSeek V4.1 Flash (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"08c078276b4f640ea4024ac62edd42bddd0ac4e10af6e08f6ae6246dc7fd2757","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.456167,"score_display":"39.5","source_stated_rank":null},"run_fingerprint":"026acafb031b04ef88d18c1ca62b2ed47308e86896c11e611aa743b2bb8069e5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"016d6820ee50295ccb08886cb582dee9bfa33c9aaa4e9d95b560d81a86099415","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · intelligenceIndex"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash-non-reasoning","upstream_model_label":"DeepSeek V4.1 Flash (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"4a3cffe5e93c6134732d2aba2f06b3b071ab18dc4144b4e69cd17aea7284c80a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.673359,"score_display":"24.7","source_stated_rank":null},"run_fingerprint":"ab6a23d34a391a4a30ad32d7dc8a01ef498c0735d1e143a6c84dd5f6b31e6f48","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_965be53371d867e0285cc728","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"68517eb7b6e4b5b3e57e22deae601143f31f314648cd86f10d39a473d4d219f8","metric_details":{"license":"MIT","variance":12.496437193891838},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1481.368809675447,"confidence_low":1467.5117466553513,"sample_count":8728,"score":1474.4402781653991,"score_display":"1474","source_stated_rank":38},"run_fingerprint":"cd1b9c6cc2f58a925b9fe6c237e46b82c216ebf2194a974267e175687480b410","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-10T16:33:51Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5d27d3306736e6ea323f70ba2ccbe5d357ea98b7dbe32c97325dae2a9587b49a","metric_details":{"category_scores":{"Agentic Coding":77.27266666666667,"Coding":80.037,"Data Analysis":79.25333333333333,"IF":70.03325000000001,"Language":81.18966666666667,"Mathematics":93.28975,"Reasoning":86.69225}},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.10970238095238,"score_display":"81.11","source_stated_rank":9},"run_fingerprint":"09bea05367537ef885513ab7f085a77ff0aa099054e84c9f36dc84582fbddb53","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":25,"at":"2026-09-10","basis":"evaluation_at","evaluated_at":"2026-09-10","first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b94f49b2a7f17debd460c998ede46067b772c5f476b47c8cdcb4915a2c205ba9","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/deepseek-v4-1-flash","source_model":"DS-V4.1-Flash"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6947fd6a3208c0e07be0853cb9fc9857d705337e57801959fd2176d772ec06da","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":36.8,"score_display":"36.8","source_stated_rank":17},"run_fingerprint":"9eef60794805d1e4afad57748ded34bb7f91de9cb61e52085810b47469fc2e50","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_deca33972debf2b7e8979c5c","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":25,"at":"2026-09-10","basis":"evaluation_at","evaluated_at":"2026-09-10","first_observed_at":"2026-09-10T16:33:51Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"89a4fe1ed816637e31a5363f4ea92c91d154a7edef1f00f4b70c393a7bb2a656","metric_details":{"modality_lane":"not reported","notes":null,"num_parameters":763205315794,"pull_request":29,"result_file_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/resolve/refs%2Fpr%2F29/.eval_results/DeepSeek-V4.1-Flash.yaml","source":{"isExternal":false,"name":"DeepSeek-V4.1-Flash model card","url":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"}},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":36.8,"score_display":"36.8","source_stated_rank":31},"run_fingerprint":"8604f980340869ad553559f9a0546c30b072754e85b38c112c41266842b4006d","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_493b765c96d952a54626d52a","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":25,"at":"2026-09-10","basis":"evaluation_at","evaluated_at":"2026-09-10","first_observed_at":"2026-09-10T16:33:51Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"97f9f7fa6748231da600827c02ed33669ea6d7a462f543c6bf9586d3c70af19a","metric_details":{"modality_lane":"not reported","notes":"With tools; harness not specified in the model card.","num_parameters":763205315794,"pull_request":29,"result_file_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/resolve/refs%2Fpr%2F29/.eval_results/DeepSeek-V4.1-Flash.yaml","source":{"isExternal":false,"name":"DeepSeek-V4.1-Flash model card","url":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash"}},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.9,"score_display":"63.9","source_stated_rank":1},"run_fingerprint":"226d1afe25ab4add2f7a5a6a371657a00418df71319784f340703088b39b507d","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_493b765c96d952a54626d52a","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1a4a5d87663dc8477a8bdf02177f86103652f82a6574eb76db742410b69294b4","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · hle"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash","upstream_model_label":"DeepSeek V4.1 Flash (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a91d506bd54adfb2713a4f212c5a31c1f083db1bcc9aff2d0b13f42c6f18641c","result":{"confidence_high":41.326926,"confidence_low":37.209892,"sample_count":2158,"score":39.249305,"score_display":"39.2","source_stated_rank":null},"run_fingerprint":"6e10d3f8220224886d2692e8c07b8ef3b749917b4b9f2250998aabdc652a78ed","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0be9f30548f55121ea9b53666b7eb5d2c692acab10d655ca1c2b96dd02c1f4c6","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · hle"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash-non-reasoning","upstream_model_label":"DeepSeek V4.1 Flash (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"56778c8e49b733dbf9f461527f28cb01b57a44e2844830bbab9639df0381b869","result":{"confidence_high":12.176787,"confidence_low":9.556609,"sample_count":2158,"score":10.797034,"score_display":"10.8","source_stated_rank":null},"run_fingerprint":"55a4c954bccb8cec89a72e4ae956ea894fe6c45e3ae3dc5c7e61c8fdcb227287","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_965be53371d867e0285cc728","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c98f8f51584a6b8b700a1f4936e355b3e70814da86fb603b3d1f8018762b9fdf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · mmmuPro"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash","upstream_model_label":"DeepSeek V4.1 Flash (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"300c68c498458b54e8f9a5f3bef6cd75b26ca6bac101261feb89e4a8ab219284","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.99422,"score_display":"77.0","source_stated_rank":null},"run_fingerprint":"df56432788985322b4adb210ee771940dd2d434c671a15f6b6484161e87fa804","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9030888006a53245a63b528945da198fe99d525356f7fd0fc9b6952e38032d67","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · mmmuPro"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash-non-reasoning","upstream_model_label":"DeepSeek V4.1 Flash (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"de224729c24fcfd53846fe4aed7d810b96b54d907d79fdefad2c7e8db53e7ae7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.606936,"score_display":"65.6","source_stated_rank":null},"run_fingerprint":"4fcef43354f3af936774be15e425488baaeb678321ac8bb4d34083dcc1db6ffc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_965be53371d867e0285cc728","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"54fbc00d0add111e18cb04f188d32c4031f37c479c2c53ef1c8316842c615852","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-09","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4.1 Flash (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4.1-flash_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"69d60d2e51b434f80742134cf07c7ccc7c730aa60c1a1a99ee9e359fd286bf39","result":{"confidence_high":24.25608,"confidence_low":7.981759,"sample_count":71,"score":14.285714285714299,"score_display":"14.3","source_stated_rank":80},"run_fingerprint":"48b30e7bd65d5c97c03148793be241e2675c44deb329c9c96cfcbd72d623b6dc","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e0eaf4614423cca2588de36fa180b2160e4dbf1750b563d1144f37892c3bcbf4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · critpt"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash","upstream_model_label":"DeepSeek V4.1 Flash (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c8a17ffa5b5f6aa2aa1ac7c82241ee7cfdba1d06639af130135ef44c07f776cc","result":{"confidence_high":24.338634,"confidence_low":7.948856,"sample_count":70,"score":14.285714,"score_display":"14.3","source_stated_rank":null},"run_fingerprint":"80415bcdcb2838ff6dda674a87a58dfa9d50310dd88f088fb3e108260fdc0344","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2db204a5f11d112c7b3eaed5552f357cd58cb0f763afa20a0387303e2707c572","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · critpt"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash-non-reasoning","upstream_model_label":"DeepSeek V4.1 Flash (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"4ee47c42fbc1443005c7d7df64da3604fb9fce9a7505a9bc03180292748ae511","result":{"confidence_high":5.730683,"confidence_low":0.013504,"sample_count":70,"score":0.285714,"score_display":"0.3","source_stated_rank":null},"run_fingerprint":"cfa253a1c4ae624c4df9244d355ec7a2e23e1734430a558e52470b83432fb7d1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_965be53371d867e0285cc728","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-live","metric":"resolved_rate","name":"SWE-bench-Live · Lite","release_id":"release_9d2de9d5be7e9822af19f7b3","split_or_window":"lite-300-python","unit":"percent","version":"SWE-bench-Live Lite"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":8,"at":"2026-09-27","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:18Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-27","status":"fresh"},"measurement_id":"1fee8f02ac12315792e61b37a4d5762969f16686cbac3ecb838129ab7e51df59","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 300 scored items with a 95 % Wilson interval.","leaderboard_script_url":"https://swe-bench-live.github.io/leaderboard.js","publication_status":"cleared-with-attribution","resolved_instances":204,"submission_url":"https://github.com/SWE-bench-Live/submission/tree/main/submissions/lite/tianxicode/deepseek-flash"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"TianxiCode","report_file":"reports-0605.jsonl","report_set":"lite","source_model_label":"Deepseek-v4.1-Flash"},"run_count":null,"scaffold":"TianxiCode","tools_allowed":"agent-specific coding tools in the official SWE-bench-Live harness"},"protocol_fingerprint":"d61dc76751ffc5acc0426b88635a8c235020986766903d39e275b088307347ba","result":{"confidence_high":73.022551,"confidence_low":62.522285,"sample_count":300,"score":68.0,"score_display":"68.0","source_stated_rank":null},"run_fingerprint":"beb962bd9328dc8cb5a6a691611403d820672e22522739360c6ca534dbb3b4ee","source":{"content_hash":"e0226b6333c547885c15c6d53074d0ad5e2a7b99db2d6b54640d7d6537c242b4","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-09-30T18:53:18Z","homepage_url":"https://swe-bench-live.github.io/","id":"swe-bench-live","last_fetched_at":"2026-10-05T12:33:28Z","license":"Benchmark and harness MIT; aggregate leaderboard results redistributable with attribution (maintainers' confirmation 2026-09-02)","locator":null,"name":"SWE-bench-Live · Lite","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-bench-live.github.io/reports-0605.jsonl"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_58c877d1022e326e6412c759","provider_config":{"source_id":"swe-bench-live","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"68265397e2f591a9fe8879a1b4e2fd33c2e789f26a316624a5c9bf54295db7a2","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.24255,"date_is_proxy":true,"latency_seconds":1109.577,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.557},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":384000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"high","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4.1-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"023f187b4c60ed4796346233c67b180e6cd5ed2df88a2d86ed10dea0a92ad1b9","result":{"confidence_high":45.289719999999996,"confidence_low":35.26628,"sample_count":null,"score":40.278,"score_display":"40.3","source_stated_rank":31},"run_fingerprint":"b083690cbcaaf7ff74a16a795a6a251a13b1a209cd1aa703b77502ee35470e19","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_acbed0464c627a0050ab0f55","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1617b1f2200c842d182165ef3e1743f53b326969dbb71c5fec2db2c1f1eb9af0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4.1 Flash (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4.1-flash_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fc3067b1b4a2cbe74698f55ae61b0267526b3d6addbe1c9e8a12a15083350e59","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.851851851851805,"score_display":"51.9","source_stated_rank":73},"run_fingerprint":"b4bc2c8d91ec2774a06bcde20d5bc97af39dfd9b6e108bd66b621e25d8570c20","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5b0c7f0a0f6f948122ebc7a94b398330585316177275d8625c6e3c683d443ae0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · scicode"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash","upstream_model_label":"DeepSeek V4.1 Flash (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"58253e3b948d3ddbbae7d7c57ecf1cf4408348283653ba674f44aa15b84146c6","result":{"confidence_high":57.560164,"confidence_low":46.094787,"sample_count":288,"score":51.851852,"score_display":"51.9","source_stated_rank":null},"run_fingerprint":"17675d978988ea9de8f40cedc6d1c920cf6b14d6e02228da2a310562cb6e66a9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7b4cf684a892deb763ba9dc542fcbee48801c7c6a351dffad4d05df2e17d0ca6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-09-10","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · scicode"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"deepseek-v4-1-flash-non-reasoning","upstream_model_label":"DeepSeek V4.1 Flash (Non-reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"97b54da2e1a5ccb038fc6e9c8281fcfb7327eadd1c84efceb6b6655fca417a48","result":{"confidence_high":41.217328,"confidence_low":30.228369,"sample_count":288,"score":35.532407,"score_display":"35.5","source_stated_rank":null},"run_fingerprint":"3d563ad245ee92cc76d4bffe5eb325286b00783394565d615cad91e820f999ea","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_965be53371d867e0285cc728","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"28734c35017b654ee27f93051f88e69e4c32d8097bcb7004071f5bbad2f5bc8a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.081663","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-09","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"deepseek-v4.1-flash-max","source_effort":null,"source_model_version":"deepseek-v4.1-flash_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"72f06cbb6509a75a6886572ca0d438e3c26b92729b2fb1be0ff3c8e5a5e98e30","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1092.35,"score_display":"1092.35","source_stated_rank":44},"run_fingerprint":"15da093eec3b542fdeb72068ff2ffa8cae54e1ddfe2b67b329e3d02637a35223","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0ab8ca57b122a5606ea0444aba52f638d94ef3951cf97e314ca2f82a5c6e59db","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.407445,"date_is_proxy":true,"latency_seconds":928.101,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.891},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":384000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4.1-flash"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"4c65daad6dc61c344f850bb7d7f5857b79e08865daa4954ff78d613332f7dbc6","result":{"confidence_high":90.40536,"confidence_low":79.07264,"sample_count":null,"score":84.739,"score_display":"84.7","source_stated_rank":14},"run_fingerprint":"32990f42c15acbd56010b253b415bdec39f1068b470db80ba92f9bba0ff7c8c4","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_acbed0464c627a0050ab0f55","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"93357cf5b6deff2162e99e83d3e1f1add4796fea472dfa430a6608e2115101f8","metric_details":{"license":"MIT","variance":28.39465480555485},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"deepseek-v4.1-flash-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8fc5103a667282fe1ad2bf34a00a64006bf2c0fb42a866e70c0603c2a7863536","result":{"confidence_high":1630.3587727135457,"confidence_low":1609.470795581869,"sample_count":3960,"score":1619.9147841477075,"score_display":"1620","source_stated_rank":21},"run_fingerprint":"5a1f1196bd17ec4e9aae0b3b25070a473e26d2a07576c101215e2b684f07d4c4","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3592cdf142ee187701704570fdc675770a0569a379b7fc053546ca91df1ed0ee","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-09","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4.1-flash_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"e80567596a8a54dc9ee62907c3a08b05306845c1e232cde8949d74c395f3f1bb","result":{"confidence_high":1631.08,"confidence_low":1596.82,"sample_count":1361,"score":1613.95,"score_display":"1613.95","source_stated_rank":19},"run_fingerprint":"8f5f87c7a7c813fc6ac9202afbc024b7305b0a1056d27eebd934b0d6ed9e305a","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9fb1efba3ff134747e4779ecf458cf78b1380457cede9fbe0eb894b53e6b437a","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.098877,"date_is_proxy":true,"latency_seconds":604.506,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.633},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":384000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4.1-flash"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"fb59c99ff5694991ab41367cd71caba4fd9930285ba54d832f35f96d17d8697c","result":{"confidence_high":77.73268,"confidence_low":71.33131999999999,"sample_count":null,"score":74.532,"score_display":"74.5","source_stated_rank":18},"run_fingerprint":"cc2b59ea81c205940c3c06fd4190cdcbcef4e8720bf9fd49026985e66f8c2149","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_acbed0464c627a0050ab0f55","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8659d03174781415d2d27709af5f3ec968d04356fec4a3f15e8a33ece45f7153","metric_details":{"confidence_level":0.95,"license":"MIT","session_count":112938},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Deepseek V4.1 Flash (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2fa7660accf1ae534b67a72ce826fd01275d178832f5bc4ec35159f7feae9181","result":{"confidence_high":0.044915087111267134,"confidence_low":0.0355845372330827,"sample_count":16932092,"score":0.040249812172174916,"score_display":"0.0402","source_stated_rank":17},"run_fingerprint":"24fdf52ac045d5750a8c97646de1b44f535aeedeb872188fc88cf598edd107a5","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"070738c36e7f382c0886d73a4e99b098778081b89ab0dc0a988dd9dd74ebf3cf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-10","reported_confidence_interval":"-0 / +0","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · gdpval"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"deepseek-v4-1-flash","upstream_model_label":"DeepSeek V4.1 Flash (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"6aed3ac4cf60d775a08178e39bb9c891bd3fe90ec3dddf823ee39001e87f0341","result":{"confidence_high":1600.0,"confidence_low":1600.0,"sample_count":220,"score":1600.0,"score_display":"1600","source_stated_rank":null},"run_fingerprint":"8a5e2f871358e2d52a9df7b216ec1dcd729ecbe0c0e196914c630fa724a184c5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_f5050027bca49136f8c18c5b","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e74de029a13984d05236c91a239e757204a9acd4dcad0920b4e06774fe3a898c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-09-10","reported_confidence_interval":"-19 / +19","source_locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · gdpval"},"model":{"id":"deepseek/deepseek-v4-1-flash","name":"DeepSeek V4.1 Flash","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"deepseek-v4-1-flash-non-reasoning","upstream_model_label":"DeepSeek V4.1 Flash (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"5222de5fc87627de16137a127a15c24220b98de0466d1d2cc9df657bb6a548e8","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1327.69,"score_display":"1328","source_stated_rank":null},"run_fingerprint":"bdec5ab60e31fabc91d56e13cede9c006935424959230c18da26913224cf9da1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4.1 Flash (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_965be53371d867e0285cc728","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f653264cbcb7ac9d0e97c565cec0a3e7d82147ce2094dade1523e1e58612b137","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-08-05"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Muse Spark 1.2; model release: 2026-08-05","source_accessibility":"API access","source_model_name":"Muse Spark 1.2","source_model_release_date":"2026-08-05","source_model_versions":["muse-spark-1.2","muse-spark-1.2_xhigh"],"source_reasoning_efforts":["xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8e3f11d46f4865fd2f0d4d73eeeef6f8b74d9321a420186069de77ef9accd605","result":{"confidence_high":157.47,"confidence_low":152.8,"sample_count":null,"score":154.99,"score_display":"154.99","source_stated_rank":30},"run_fingerprint":"713a3dfc36f2955dcaa3367c1e47ec0660a969bee8b499530909313ab2ecbe53","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_07d1816d63900bef656c0e24","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fe8ac19cdada067decd9e6cd9ed6a22dd273fde5e58e48d1b36219619d0cd52e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1048576,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-05","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · intelligenceIndex"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"muse-spark-1-2","upstream_model_label":"Muse Spark 1.2 (xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"9815dc70852978a4a0bc8e91ee41487753e181e702fd4c0d1b7c58ea0912cbd2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.803911,"score_display":"39.8","source_stated_rank":null},"run_fingerprint":"b1445207cb9a198dc118cb66d29169f768bbabc0f390ad35e3b68a34eda9db34","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f517ee2b762e564e78a73710fe3a336811444216836461cca72b28846a6d9c21","metric_details":{"license":"Proprietary","variance":21.645464392265332},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1502.630581066812,"confidence_low":1484.3932391423223,"sample_count":4356,"score":1493.5119101045673,"score_display":"1494","source_stated_rank":11},"run_fingerprint":"c7629d716beaf11358b3eb2d55f3527b7c41528946dc1ddea210ab6a90305e04","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fef56c3882cb3a5c87f853094db8aa5b70bcbe29830af4469779025d09aad31b","metric_details":{"category_scores":{"Agentic Coding":57.57566666666667,"Coding":77.5415,"Data Analysis":76.45833333333333,"IF":74.325,"Language":78.57466666666666,"Mathematics":91.20349999999999,"Reasoning":90.00475}},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.9547738095238,"score_display":"77.95","source_stated_rank":20},"run_fingerprint":"e19cc7a77d815324ffc30e5e0808d35a4d3eabebca1174c9451499a5c762c398","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dcb8d4cc52cd7833dbd677241d88ecb7118c567cf92678086affdccfef13cb61","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-05","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · hle"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"muse-spark-1-2","upstream_model_label":"Muse Spark 1.2 (xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b9554e2032c7be8036d239d362912f1ce3541c0223e284819304e8efaae5e25a","result":{"confidence_high":47.565858,"confidence_low":43.367798,"sample_count":2158,"score":45.458758,"score_display":"45.5","source_stated_rank":null},"run_fingerprint":"554e935bd6f3b4fa060b4263f50fecc07f64f16dee7eacda35ac3772edacc6cd","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:36:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:36:49.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"be10d393bfb6b0870de7e10d1a7569bc678c1149e1f0bc54eaed994474d52414","metric_details":{"best_score_across_scorers":"0.6030150753768844","model_release_date":"2026-08-05","stderr":1.5518791563606453},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mLLmVVvo6t23MM5yxq7ryW","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmLLmVVvo6t23MM5yxq7ryW.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mLLmVVvo6t23MM5yxq7ryW.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"cf5cdcc4c3a70b7351aabe6cbba24f9eca000dac537b1ffcdeb66a58b7031afe","result":{"confidence_high":63.3431906841553,"confidence_low":57.25982439122157,"sample_count":1000,"score":60.30150753768844,"score_display":"60.3","source_stated_rank":15},"run_fingerprint":"4cb4d6ccd4f564accb5d7b6e508297b1b7146630844123ad6193eaaa5f59a99e","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e19fdf92fefcf3881fb4679842a2bec953f91cf7a82539d4787b035162342db0","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-05","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · gpqa"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"muse-spark-1-2","upstream_model_label":"Muse Spark 1.2 (xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"87f968a546c9523fe5682c528ae62c619eecf75dda377913fd1ef5d1055eeba0","result":{"confidence_high":93.77056,"confidence_low":85.49952,"sample_count":198,"score":90.40404,"score_display":"90.4","source_stated_rank":null},"run_fingerprint":"fa240e3bfe3d08d0494982d85de1d7f742ea7b7a4518d5ea86b8cda5c37be6a1","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9cc69483b57f57a1d5b49366f43a0e0261cd22b410b269d516cf70086771b844","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.010668,"date_is_proxy":true,"latency_seconds":44.234,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.318},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":131072,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"meta/muse_spark_1_2"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"77e3c40eea0d5a45aa05d9eee71f8a2f178e0a642f2a846a514b8f45027b5373","result":{"confidence_high":88.89828,"confidence_low":87.65172000000001,"sample_count":null,"score":88.275,"score_display":"88.3","source_stated_rank":20},"run_fingerprint":"625be949b3958657f938a6b1bb2730476dd2eb3f6ed970f9d90486eb4723f553","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f553916c4e1403d6c605f464d49c10557b722a06e40596ba2cd2ee47a47948e2","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.018022,"date_is_proxy":true,"latency_seconds":73.839,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.831},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":131072,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"meta/muse_spark_1_2"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f76ce79e21835158a359f1b40342855b4557a1e43deefaf733c60439e13bd751","result":{"confidence_high":87.75576,"confidence_low":84.49824,"sample_count":null,"score":86.127,"score_display":"86.1","source_stated_rank":22},"run_fingerprint":"6a05bc8cb6980fa03d945b79b11a83667506551c7ea36817eb21b5c047be6006","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"534e3ff526dc6f458a4e90742e17c7e2155e341b91d6d67e91d6fdce746aed3d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-05","notes":null,"upstream_source_url":null},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.2 (xhigh)","source_effort":null,"source_model_version":"muse-spark-1.2_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"56a6f69c84b56661f156008c5e12abc0d521442f69d1112d593bae78fab30122","result":{"confidence_high":28.178675,"confidence_low":10.564331,"sample_count":71,"score":17.714285714285698,"score_display":"17.7","source_stated_rank":61},"run_fingerprint":"17a604e35602939c6b55969461325934ffe34696468b083d12557be7dacdab58","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e561e2c180a7c6dbcc553f51e83369223e27fec7fa8c884743c23c31677dfe9a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-05","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · critpt"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"muse-spark-1-2","upstream_model_label":"Muse Spark 1.2 (xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"f55d9c93625d044456b083c64e02b796faf883c0abf95ad21b32ebb2bd9a6221","result":{"confidence_high":28.262679,"confidence_low":10.525214,"sample_count":70,"score":17.714286,"score_display":"17.7","source_stated_rank":null},"run_fingerprint":"ae15a23dd8189aac47c11123b1a2c349cc6cd7d0ee05674f04c0fe4251eed660","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b7f6ef3c2119e8bc598c56e09d8ef387a6d895aecfeb3d849c44a79c337556ea","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-05","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"muse-spark-1.2","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"84da92f7c5c07b1b3e560be218e49d78ebd66a5ca25cdcb81951f69bbdb448f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.5,"score_display":"74.5","source_stated_rank":9},"run_fingerprint":"a34ec3793b72324574c7c6323129a230cb626b2c9b08c46cdd87bc32b30c4079","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_01969cbf5dae3b1269d94614","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2d5e88713b8117384602728db95fd8589bc1180e1f2549796a68f4e6f67b33ee","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.554231,"date_is_proxy":true,"latency_seconds":487.088,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.525},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":131072,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"meta/muse_spark_1_2"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"b57135713ed5106e99f0a2f11f7efdab02a6b908cc108d91600df27771f94c35","result":{"confidence_high":89.589,"confidence_low":83.61099999999999,"sample_count":500,"score":86.6,"score_display":"86.6","source_stated_rank":13},"run_fingerprint":"59469052249525dc8d98a9dfe24ab5a4a054c87d2c0817c00c04bb0f7361692f","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"e3d8e74aad1585cf832823e02c0f2cbd6e414cda49c90a5544288226891aef24","metric_details":{"ci_attempted":452,"ci_half_points":2.124297201927127,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":248,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":100.75884955752213,"mean_cache_tokens":12584988.949115044,"mean_cost_usd":3.6955048180309733,"mean_duration_seconds":927.8692516349557,"mean_input_tokens":13693824.435840707,"mean_output_tokens":99226.38053097345,"median_agent_steps":94.0,"median_cost_usd":2.6840506,"median_duration_seconds":784.787489,"median_input_tokens":10339991.0,"median_output_tokens":81250.0,"median_output_tokens_to_pass":77863.0,"median_peak_context_tokens":189481.0,"n_attempted":452,"n_passed":248,"n_tasks_attempted":113,"n_tasks_passed_any":92,"pass_at_4_points":81.41592920353983,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_muse_spark_1_2_xhigh","source_model_version":"muse-spark-1-2","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"28282cf783bafcecd7f7d9d45b49a4ca5f96b889be7c9be1937e83965b47aca2","result":{"confidence_high":56.99155383909527,"confidence_low":52.74295943524101,"sample_count":452,"score":54.86725663716814,"score_display":"54.9","source_stated_rank":39},"run_fingerprint":"6f36446b918ecd4442e0c62dccc20ec6009e5cb2005585e6294c340871628c66","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f3ac85e36de290e0bc88ab483ead13c7149e4bbfc6dccc23222ab069dabfd082","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"100.75884955752213","mean_cost_usd":"3.6955048180309733","mean_output_tokens":"99226.38053097345","model_release_date":"2026-08-05","notes":null,"pass_4":"0.8141592920353984","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"muse-spark-1-2 (xhigh)","source_effort":"xhigh","source_model_version":"muse-spark-1.2_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"f70ed800e17ff1733f91df704d4529fb265348941ea27326f7f1c81480f3a4f8","result":{"confidence_high":56.99155383909527,"confidence_low":52.74295943524101,"sample_count":113,"score":54.86725663716814,"score_display":"54.9","source_stated_rank":38},"run_fingerprint":"9511783a8e13d287cfe571f8857950b15120a4b42e8c0b364281b528886317be","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0407493aa6bba78c74cce27d7a7cf543d610d98f75366bfd7e179a1e402324f2","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.640493,"date_is_proxy":true,"latency_seconds":1534.626,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.873},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":131072,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"meta/muse_spark_1_2"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"04a2b48d19ce42b1973540cebd759a2e917e68cc51934cf0fc0371abd564000a","result":{"confidence_high":23.489079999999998,"confidence_low":20.06692,"sample_count":null,"score":21.778,"score_display":"21.8","source_stated_rank":37},"run_fingerprint":"323d65c0b199f6386537a38b95a5cca9a54491c9d7ee0d5a26dfea87dcefe830","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1f9683ea3d985aa9f09a73960d3f0da8bd9c6706a4f0780f68eda6d0bc8f791c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-05","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.2 (xhigh)","source_effort":null,"source_model_version":"muse-spark-1.2_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"bb0943080f19f9e7003f62cb552b20e4da420086ab9e8f74b257f5f7c01e7629","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.3657407407407,"score_display":"56.4","source_stated_rank":31},"run_fingerprint":"7c09c99779b1808435da0219c4f6a894453fe5fc822542b757e74e697764a05b","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3e73aaf2da106ddfeb59d6d58bc29e4a8c8188f3e225d35b6e10568b596086da","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-05","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · scicode"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"muse-spark-1-2","upstream_model_label":"Muse Spark 1.2 (xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"75de278fa6c4dfa97d389df33637dbe9963b3f9dfa601b168f5d96fbca4672b8","result":{"confidence_high":62.984009,"confidence_low":51.635793,"sample_count":288,"score":57.407407,"score_display":"57.4","source_stated_rank":null},"run_fingerprint":"397cf713a20c8156d5672e3defd0895660081a7933ddd0bf8d52bf6cd9a67d21","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6f702c13e1b645137c271f41c0e0e57c2373f436f061bbb1f746801e1587191c","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.529642,"date_is_proxy":true,"latency_seconds":1195.221,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.305},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":131072,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"meta/muse_spark_1_2"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"4197dfa4f2c4bc97a9ec9baf28a59f923787e9a78721682bb7d33a41c43ca0ff","result":{"confidence_high":85.5778,"confidence_low":72.62219999999999,"sample_count":null,"score":79.1,"score_display":"79.1","source_stated_rank":22},"run_fingerprint":"fe91a18baaa4ffe92f2df2bd588bd870020e86d3f26dedfe73591191012f6136","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"360c586f242ff5712f67c4977774e2e90000a5ec53bcb5e99f7f1d72150c0bc3","metric_details":{"license":"Proprietary","variance":48.08852461010785},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"muse-spark-1.2 (xHigh)","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"167902f666244faf032b5b6589a59a41bb2cfcd9620ad7b8bec592954bbaf160","result":{"confidence_high":1545.370094476105,"confidence_low":1518.1870050977677,"sample_count":2183,"score":1531.7785497869363,"score_display":"1532","source_stated_rank":44},"run_fingerprint":"a6a56c2688f5b635764ec08c08ee1481e4bc674ac9a00adf06de4164847f1460","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2af00d631361802268e32bc8bbc968c82dbcfb448473737aed8989fee364c8a9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"muse-spark-1.2_xhigh","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"f081ee356738bbf3dee9c0ce6f10d1d3c239ba10ac3f17d9adb34a0b65369da8","result":{"confidence_high":1548.32,"confidence_low":1520.43,"sample_count":2065,"score":1534.38,"score_display":"1534.38","source_stated_rank":41},"run_fingerprint":"5e7ed47a685cd435e62cf5ee66d12088023cac8731ad57035cd16efc793f3bd8","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a4f0e73c0e77cb1c5cd280193013cf1c00cd51d922ebd60ecc36b51dea803bf0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-05","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"muse-spark-1.2_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"40925098be8d19a7ea155e1d80d593ef09345a8b8ceec661dc3c16dd00655b43","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.3,"score_display":"60.3","source_stated_rank":51},"run_fingerprint":"4eeee13353a483934aa19071735fd1d797b9b13e6eb88a34611344e31d4f8e18","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"79795e767d43f6b9f6dc432b32f8c913f7beb3dd9ba646e9bcb52feb2fc1b64d","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.499472,"date_is_proxy":true,"latency_seconds":879.273,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.649},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":131072,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"meta/muse_spark_1_2"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"96570fef9d566fade96aa67dca00939b8c6aa7c7ac5633241e0e80f72448b26c","result":{"confidence_high":70.93504,"confidence_low":68.39095999999999,"sample_count":null,"score":69.663,"score_display":"69.7","source_stated_rank":28},"run_fingerprint":"3c78450acbe2db21e7eb6d5f12bc0aa03fc366dabf17651af46ed334012158f0","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cbc614007ee846975a8927c3726d9d946fe3fcb3f389b4a9ed3ea671380e2f78","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-05","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · terminalbenchV21"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"muse-spark-1-2","upstream_model_label":"Muse Spark 1.2 (xhigh)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"fb29dafae87e099e54f9c94e6d2e8dc8381824092ca08dbcbf0be503faeee3ba","result":{"confidence_high":87.111311,"confidence_low":70.693237,"sample_count":89,"score":80.149813,"score_display":"80.1","source_stated_rank":null},"run_fingerprint":"744d9d7e45bb4ad735518fb56b0deea2ef9c0d93b58b0a246a9612431d87e004","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bf319456418e5eab8284dea263063a81d0a7ae416081a6c20102c340e8381ce2","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1048576,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-05","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · tauBanking"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"muse-spark-1-2","upstream_model_label":"Muse Spark 1.2 (xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"60a5c6f1fbabc90023fd5a9c18df1c000d431d1e29a2930a21405eeda84e113f","result":{"confidence_high":44.740551,"confidence_low":26.104815,"sample_count":97,"score":34.845361,"score_display":"34.8","source_stated_rank":null},"run_fingerprint":"48b525abb074718ab62c6b1902ed78800280815b08d0ee5028886f67ba4add02","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.2 (xhigh) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"372f26c4d0b3e7df8526c21ee122f7a1321337709efe1442707f258cb234a5e9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.517","mean_standard_error":"4.9","model_release_date":"2026-08-05","notes":null,"standard_error_points":520.0,"upstream_source_url":null},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.2","source_effort":null,"source_model_version":"muse-spark-1.2","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"dbfed7e5c15cc2ba935721aaf883226036efbae07680a0e352c41aeb280dbb30","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":36.4,"score_display":"36.4","source_stated_rank":35},"run_fingerprint":"ede79e0711d0e03969e1b13e6bcaa0362db8f79ccb01b44e9ef3a55f018a37b3","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6949cf33fc5bd1ec3a3284a7","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"578a43381c7382185b0259cea85bbf117ec6da9d8600f80250d51a767e2f6c8e","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":48150},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Muse Spark 1.2 (xHigh)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0711fc99b7f486eff0f2dfd85523168e3cfbeda389c52a8d0297ea859864f88c","result":{"confidence_high":-0.025990783449722877,"confidence_low":-0.03946558677148422,"sample_count":2884900,"score":-0.03272818511060355,"score_display":"-0.0327","source_stated_rank":38},"run_fingerprint":"2ef393e84e2c9c55b069ece4d8cf7351d4288d21a1c73f767bc0330f9865cea1","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"954dd164fdcea03a6800d1b750be0ed39c86166181f8a273a1b32f4570309137","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1048576,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-08-05","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Muse Spark 1.2 (Xhigh) · gdpval"},"model":{"id":"meta/muse-spark-1-2","name":"Muse Spark 1.2","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"muse-spark-1-2","upstream_model_label":"Muse Spark 1.2 (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"ae74fb425a48e64d4aed977c9def52526fcc6583f0f45b96c13f16a504426330","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1481.62,"score_display":"1482","source_stated_rank":null},"run_fingerprint":"c50b4b16831b1929ee8ea3fe4a3a437608e895f9a9303510e069acac02eff01a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.2 (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_502229b22892e7bba82f7d86","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2073cb4dcc5152ee91064eaa171bdc798822c993dfba9196cfe12e97242c8732","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-19"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 3.1 Pro; model release: 2026-02-19","source_accessibility":"API access","source_model_name":"Gemini 3.1 Pro","source_model_release_date":"2026-02-19","source_model_versions":["gemini-3.1-pro-preview","gemini-3.1-pro-preview-customtools","gemini-3.1-pro-preview_high","gemini-3.1-pro-preview_medium"],"source_reasoning_efforts":["medium","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"23d23e96ca6d8df6a313ff65a8b02f6cac09e182a4a0a6a79c3d4e682c68b971","result":{"confidence_high":157.34,"confidence_low":152.58,"sample_count":null,"score":154.84,"score_display":"154.84","source_stated_rank":31},"run_fingerprint":"94663a56ff95b98f6f4b4106822c495d16018c274dfa2dd347b1e46ae5ca706d","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_77bb122ce308b85a06f0e2bd","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"56c2640eb6c06f4f2c16cea17efe3cdd74907b988ab5fe146dba6d4ab2ee7cd0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-19","source_locator":"Leaderboard models table · Gemini 3.1 Pro Preview · intelligenceIndex"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-1-pro-preview","upstream_model_label":"Gemini 3.1 Pro Preview"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"49682aa4875938f34e3027c2a40037d3f5207447df39082b7b3a14db3fd11767","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.718566,"score_display":"29.7","source_stated_rank":null},"run_fingerprint":"74f8fe5bb73dc67cb22ad8e9672c95feecb307b98c6602cfb2b274b7097835ee","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Pro Preview · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bf7a013a6ba02a14d73d95c","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9319b4189e4c9d8e4bda53f8411fde930e92f2a3fc00c79b6735d922f38f616f","metric_details":{"license":"Proprietary","variance":2.3589385464609713},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1489.9951678963012,"confidence_low":1483.9746146703,"sample_count":121806,"score":1486.9848912833006,"score_display":"1487","source_stated_rank":18},"run_fingerprint":"d071895802bef1adcc737240492179b62e3177c1ca7e2c697ea2c648565f2c8d","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fb779cfb20c726fd0ba130d3","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"cb19d160cbfae2bc43c8f306b1bd4fc025f0282c357d516b34e49401940ed99d","metric_details":{"category_scores":{"Agentic Coding":44.141333333333336,"Coding":76.4545,"Data Analysis":78.54133333333334,"IF":79.1,"Language":85.37599999999999,"Mathematics":91.04499999999999,"Reasoning":84.00475}},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.95184523809523,"score_display":"76.95","source_stated_rank":26},"run_fingerprint":"9b554094532eb9618489a80a5a44bec4f4369aca79f88aaae34d53a460245eeb","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":177,"at":"2026-04-10T15:51:06Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-10T15:51:06Z","status":"stale"},"measurement_id":"c57c2031c12e23cdfbaca0a4274dd20b0b9234f78b6b298f7d5fddb2ffe7cc62","metric_details":{"calibration_error":51.0,"confidence_interval_half_width":1.96,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=gemini-3.1-pro-preview (thinking high); createdAt=2026-04-10T15:51:06Z","source_row_created_at":"2026-04-10T15:51:06Z"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"gemini-3.1-pro-preview (thinking high)","source_reasoning_label":"high","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f2c944ee897b1bf5b9c2351c456dd58645c1e6575c92ab3ad59c5aaab765c64d","result":{"confidence_high":48.4,"confidence_low":44.48,"sample_count":2500,"score":46.44,"score_display":"46.44","source_stated_rank":2},"run_fingerprint":"2a29a39af40be22c21bfbe763e6ec38a6783d9d04afa1a8db19686fa73877fd5","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=gemini-3.1-pro-preview (thinking high); createdAt=2026-04-10T15:51:06Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"evaluation_at","evaluated_at":"2026-02-19","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f6d1dfc93bb2a74096c9d40c31dc1846254bf07d10778035aeb9913ca5079cff","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/google-gemini-3-1-pro-preview","source_model":"Gemini 3.1 Pro"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"02f4a04c05254442b934554d84552e2b1d45383f95cc9b745982f05c0a6f664b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.9,"score_display":"45.9","source_stated_rank":7},"run_fingerprint":"96fd7cf407293524690e8169d8842a0cd1ec370cbba476949afa0b25c6312c26","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2ec6967c0417137a1436865d","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aac2979946cced551229580234a9a4aaab2633b6ebed945702c3550bcb152aa5","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-19","source_locator":"Leaderboard models table · Gemini 3.1 Pro Preview · hle"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-1-pro-preview","upstream_model_label":"Gemini 3.1 Pro Preview"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2228a8fc1e5cf2ac56e5ad06d90dc11d99cd019687c4709087dc6a0c7cc13c7e","result":{"confidence_high":49.143583,"confidence_low":44.935539,"sample_count":2158,"score":47.034291,"score_display":"47.0","source_stated_rank":null},"run_fingerprint":"1936e17aecd29cafa113d18fa2c73020d2636ccea2bd4a9789b686e0c101a35d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Pro Preview · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bf7a013a6ba02a14d73d95c","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T21:35:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T21:35:39.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8cb453dfdd7f5700f32d16be68f2f1257aba4ae149306c72b47d275338554ba1","metric_details":{"best_score_across_scorers":"0.735","model_release_date":"2026-02-19","stderr":1.3963164754810065},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"VGeF8Xac4WcSAGgSXVdEBi","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6ec618647aae19d98a2e6a010349940ff6e286be1c872a82002ffffe0cac6a6b","result":{"confidence_high":76.23678029194278,"confidence_low":70.76321970805722,"sample_count":1000,"score":73.5,"score_display":"73.5","source_stated_rank":3},"run_fingerprint":"1294febf90012fc4c4f3255b2ab5556a982e7265f55a882aeb0d2f1224ee313d","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":209,"at":"2026-03-10T00:00:00Z","basis":"evaluation_at","evaluated_at":"2026-03-10T00:00:00Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0bf3cad22c0c31fcdb1ee742958f53baa7afb3c5630fa4025f1019be9de62c1a","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.1-pro-preview","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"43e82d22f9c497485eabd10e54f1affd3c49309541620f806f6e893a9d4c2093","result":{"confidence_high":80.11268722473896,"confidence_low":74.92291806518982,"sample_count":null,"score":77.51780264496439,"score_display":"77.5","source_stated_rank":1},"run_fingerprint":"44aeb57f6e19cb5b5b503a78fe7b4e861db35694e7d382b44f2498c98a8a198c","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f9361744222ae669650437f","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6c3ee6b647a1fca50a4995ac608b3e9b1e60eba63fab6ac9e97f216c72bd299f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.081442,"date_is_proxy":true,"latency_seconds":65.76,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.046},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":95.96,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65535,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-pro-preview","zero_shot_accuracy":94.949},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"937651c86529f7207a02dd573dbafdf348c7b104d5478e69af3a5e28872afe42","result":{"confidence_high":97.50416,"confidence_low":93.40383999999999,"sample_count":396,"score":95.454,"score_display":"95.45","source_stated_rank":1},"run_fingerprint":"e37df4f492e81539715986751ba16bb03f227709b43fe7748e80a29adeed1a5c","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2d8bc25cf48b5c1abbabf4aa45c87a9c731e55854c22ded5ef2d1b11927d1afd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-19","source_locator":"Leaderboard models table · Gemini 3.1 Pro Preview · gpqa"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-1-pro-preview","upstream_model_label":"Gemini 3.1 Pro Preview"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"18e46cebbc3f593d91c1d2d6e544355d05dfaf5dccde41eba05ac6821390a184","result":{"confidence_high":96.648379,"confidence_low":89.954185,"sample_count":198,"score":94.141414,"score_display":"94.1","source_stated_rank":null},"run_fingerprint":"da7fd7df9c9c695955c0b4df8d84cd09eed1b2e15a6f9712259d574db01c4ebe","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Pro Preview · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bf7a013a6ba02a14d73d95c","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:30:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:30:37.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"069e07ee51dcff85df36c12256a67e57a1fb1bfa29e63c076281497bf9c53754","metric_details":{"best_score_across_scorers":"0.9444444444444444","model_release_date":"2026-02-19","stderr":1.6319950700767385},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BgmrBFsbsaD8rNpebv49Da","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBgmrBFsbsaD8rNpebv49Da.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/BgmrBFsbsaD8rNpebv49Da.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"5d0512533379efeea8d460ffd19b44bdc92eacb605c4c4df63909028c7ff1336","result":{"confidence_high":97.64315478179485,"confidence_low":91.24573410709404,"sample_count":null,"score":94.44444444444444,"score_display":"94.4","source_stated_rank":7},"run_fingerprint":"e8a860ad63cfcd313e140e4dbb322b857363e31510e20031cd29abc7d374587a","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":227,"at":"2026-02-20T05:01:02.541Z","basis":"evaluation_started_at","evaluated_at":"2026-02-20T05:01:02.541Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"57d11549ce032ae6222e11aefddf83638678f3aeec22c8f93b263aaa944549fe","metric_details":{"best_score_across_scorers":"0.941","model_release_date":"2026-02-19","stderr":1.7000000000000002},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"kR9F3oHBCnwQW4AiTPKwpd","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkR9F3oHBCnwQW4AiTPKwpd.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/kR9F3oHBCnwQW4AiTPKwpd.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"48ec8e06a0c15e4a36b6d8c23e96a586679a0edec668e40fd7d044d26b27ea5e","result":{"confidence_high":97.43199999999999,"confidence_low":90.768,"sample_count":null,"score":94.1,"score_display":"94.1","source_stated_rank":10},"run_fingerprint":"228bdefc78d5f5534bd0817fa40fba80269d01b5087608075330e4fc05b052e1","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5895a85d12bd7b813808c368","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef33ba36ede490a7cfe44a373fb06f137b935128514884513a0433731560255c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.027545,"date_is_proxy":true,"latency_seconds":23.912,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.284},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65535,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-pro-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d9479fad04b39b1f805de6051aa0148edf2a2ef789ddf1ec1e09e276879d6d9f","result":{"confidence_high":91.54364,"confidence_low":90.43036,"sample_count":null,"score":90.987,"score_display":"91.0","source_stated_rank":4},"run_fingerprint":"2bdd97b6b034ca1db263d77891b601f24044071098d39177d9ab979d4cd4c4eb","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T23:59:11.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:59:11.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7797f163b6e4d605ac64433312c615b416739d3d90f890f18c215e6f7e21558d","metric_details":{"best_score_across_scorers":"0.9555555555555556","model_release_date":"2026-02-19","stderr":3.1067790907534736},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"nhim2QCRgpir5cZdqrmUz3","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fnhim2QCRgpir5cZdqrmUz3.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/nhim2QCRgpir5cZdqrmUz3.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3829c5d77dce1f2b54d16d58262aa94aa7a4b31971c5e2b75494fe3c424c3f4e","result":{"confidence_high":100.0,"confidence_low":89.46626853767874,"sample_count":45,"score":95.55555555555556,"score_display":"95.6","source_stated_rank":38},"run_fingerprint":"6714545ac91ac7df411399a04d7a1ce9e26b20a4e92a40e0da9bf405f55c8c80","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":227,"at":"2026-02-20T04:39:36.998Z","basis":"evaluation_started_at","evaluated_at":"2026-02-20T04:39:36.998Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e8c35b1d90d7a1646638bd16ef03691e24d54be827a2832a7da3889abf814eba","metric_details":{"best_score_across_scorers":"0.956","model_release_date":"2026-02-19","stderr":3.1},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bXgZZgzRE3wXoSRUgrbsVf","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbXgZZgzRE3wXoSRUgrbsVf.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bXgZZgzRE3wXoSRUgrbsVf.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d2016b4b92de8f0800de6fcb080c74eaec31b0cffbe29d513338e2c4cdb81218","result":{"confidence_high":100.0,"confidence_low":89.524,"sample_count":45,"score":95.6,"score_display":"95.6","source_stated_rank":37},"run_fingerprint":"2358e3b62982e86be56be7b3cf1478b8799f01a247282d34a22ba4d9aabbe5e2","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3f7afb22954092bd2c683111","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T23:35:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T23:35:57.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"783dd3b76f1798ae5db4d804d4f2c9c49d2cfd09c726c23bd4042ba49e48de3c","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5964912280701754","leaderboard_private_problem_count":285,"model_release_date":"2026-02-19","public_example_count":10,"stderr":2.911181956525014},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"hVifTUdQZrFEYabgCRu8Qq","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhVifTUdQZrFEYabgCRu8Qq.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/hVifTUdQZrFEYabgCRu8Qq.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.1 Pro Preview","thinking":null,"upstream_model_id":"gemini-3.1-pro-preview","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"1d7ed5f466056f5a6721a1fbc62165c689ab91d37b31c4cf255aff8f823bcdc4","result":{"confidence_high":65.35503944180657,"confidence_low":53.943206172228514,"sample_count":285,"score":59.64912280701754,"score_display":"59.6","source_stated_rank":35},"run_fingerprint":"c7259e4bfac1f785c426919cbc8f570d5bfa1059fdf8229b8d163b6cb7f06330","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_87c06704555ddcf190611c15","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fb4fb5f9d7feade6883055424b20844b85fe909114a5cf06688d279842a8f7b4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.9622,"model_release_date":"2026-02-19T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"","model_type":"CoT","upstream_model_id":"gemini-3-1-pro-preview"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b51bdc08410f3edb3900c1005f74a21aa049ee81af67c033f943bc9ea06023e3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.08,"score_display":"77","source_stated_rank":42},"run_fingerprint":"11f3e5e131095822c5233300eb117f22d1dbbf3688a16c773f69327f18ad0ce7","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_124badaa245c24f82cccc0ed","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0c244e662f84d5a326edcc3d1a8b689534b645fd69a635a7907939d4c5f24dd7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.9622","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Pro (Preview","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"79583199c18c9eac7c3db52e6a9baaf6828629f81204b4b69e5bdc9d06269976","result":{"confidence_high":81.140961,"confidence_low":72.486771,"sample_count":360,"score":77.10000000000001,"score_display":"77.1","source_stated_rank":43},"run_fingerprint":"bb9d829de93c6331af180354bbbe0c7592fc436f32268aa64cb0d1ecc23d25e3","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c27df4edede426fa37564fc1","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"53939d6f0da7d2d1436132240270157908d36c9601c411696129a286c61446e6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.9622","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Pro (Preview)","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"180c2eedc09ecef95e97a6fd5c8754ebee1c2c23a9cafbf5925a8284baba64f6","result":{"confidence_high":81.12248,"confidence_low":72.465675,"sample_count":360,"score":77.08,"score_display":"77.1","source_stated_rank":44},"run_fingerprint":"d57151b28e77cf5ac56594e59eb3ba912c3b3128ead1af6de586015c181f62a4","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c27df4edede426fa37564fc1","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_3442b9e1c78d3919dff3a62a","split_or_window":"semi-private-validation","unit":"percent","version":"ARC Prize verified semi-private"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"60fa2f0950ec8446fb582fc33b7f4a62f0151761c7d73b5965b97315f703e08d","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · ARC-AGI-2"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"ARC Prize Verified; Google Model Card re-report","reasoning_effort":"reported","result_published_at":"2026-02-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"340365f7b2cacb3227c82581dce2678298f3faea5d7ce87e415ebfc859e8ad0a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.1,"score_display":"77.1","source_stated_rank":null},"run_fingerprint":"b05a7586395069f5d0f8e900e122321630430248bc12e780df759bd18bc5a291","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · ARC-AGI-2","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_58b76c4ee41c5507274dd5cb","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T23:35:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T23:35:57.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"28fe33b73da94e5935c700d398c4458c66228239ba089f44a2d19e5d838a06ab","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.2682926829268293","leaderboard_private_problem_count":41,"model_release_date":"2026-02-19","public_example_count":2,"stderr":7.005564203095157},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"eXVWr3RBeSsWy5DAU8ed46","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeXVWr3RBeSsWy5DAU8ed46.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/eXVWr3RBeSsWy5DAU8ed46.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.1 Pro Preview","thinking":null,"upstream_model_id":"gemini-3.1-pro-preview","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"b2443e68eed4f08bf6e45e23dd0ebd1e0e724a61ee6a77c71bbc36ffa584d871","result":{"confidence_high":40.56017413074944,"confidence_low":13.09836245461642,"sample_count":41,"score":26.82926829268293,"score_display":"26.8","source_stated_rank":37},"run_fingerprint":"27ace5ebfd1c17be755ce1073c6ea46319c3b48a1902973ecdacc658aebd255d","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5baafa5d62b0d42201016d96","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c0c0963a6baf2ea0580d4cf8e4d86c71c20883a6853ce9d6981d8b32b6686ec6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.064084,"date_is_proxy":true,"latency_seconds":76.99,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.775},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65535,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-pro-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3eda131c12e726b43103c8e9f3f69f913decf17c68410e85c72b1a8ba7b74b71","result":{"confidence_high":89.727,"confidence_low":86.689,"sample_count":null,"score":88.208,"score_display":"88.2","source_stated_rank":10},"run_fingerprint":"f6aac9195d96b14fac07e7cd64ea63fde3ca7d79bab695bbcf43387449ecaec5","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cfc4f7820928154d7c26454ef8d221efc9321328d6c8cef00f1d46866a4512e3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-19","source_locator":"Leaderboard models table · Gemini 3.1 Pro Preview · mmmuPro"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-1-pro-preview","upstream_model_label":"Gemini 3.1 Pro Preview"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"610b9d61f0b83ad8fbc031123522517f15b97b37a7bbd21c4a432f77f9a35a94","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.427746,"score_display":"82.4","source_stated_rank":null},"run_fingerprint":"1546ef955998f9291dc113e85e5d747690f245b52d2d60206bd92a72052bc522","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Pro Preview · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bf7a013a6ba02a14d73d95c","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_39a5bff9a48a29bea5ce760d","split_or_window":"standard-no-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"6f09ee4fe4fb3f126f7a297290c46dd3a37f2a12a70486e93b0cef4996dd9e04","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · MMMU-Pro"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider evaluation; standard and vision average; pass@1","reasoning_effort":"reported","result_published_at":"2026-02-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ac7c8ce3feb84e3b87fa2687459ea475273ea5f28c4ea5501f8e909343cff579","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.5,"score_display":"80.5","source_stated_rank":null},"run_fingerprint":"e7abe8d5f62046fbe361f00f1fadd3c3cfa581057d9ccbb6897b22171d1b7ce7","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · MMMU-Pro","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_58b76c4ee41c5507274dd5cb","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1ba1403bad96574129d8837565d38711debe035ca75575a2a10a396ebdc5a98b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Pro Preview","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"07cf0f58661ad156b1c5f0c3c2f302c89535fdaa8f3b618c83f43f9ac66b90dc","result":{"confidence_high":28.178675,"confidence_low":10.564331,"sample_count":71,"score":17.714285714285698,"score_display":"17.7","source_stated_rank":61},"run_fingerprint":"15562c69a9d30f8d04027c76b69a3a66659190be02b7e29b4f141e38db474e63","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c0bf25fb577709877c10beab","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"32460b15d2a02e0a09042333824d84166686404f0b8750ba01804833a9b2ee1b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-19","source_locator":"Leaderboard models table · Gemini 3.1 Pro Preview · critpt"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-1-pro-preview","upstream_model_label":"Gemini 3.1 Pro Preview"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"3413f99cec3523d7e2e2462c3829b1b60550236088c77c060a2fce76e925a70c","result":{"confidence_high":28.262679,"confidence_low":10.525214,"sample_count":70,"score":17.714286,"score_display":"17.7","source_stated_rank":null},"run_fingerprint":"27f017c1c0f2cca8df49b715651210c61914bf211649a37d83ed4a046727b4e3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Pro Preview · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bf7a013a6ba02a14d73d95c","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"85eedf23fd748adeadc0ea54cccf71f9724d4d0b929339aad29cced8414187fe","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"df2016c95911d075da9dcb44503a38e4051fe0cd9bfc78ecb819c4b5c28303cf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.6,"score_display":"79.6","source_stated_rank":3},"run_fingerprint":"d0b3613437c94347b558424946353fd1bb2842d03342505ca5d48a255afdc9db","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ff8f9f6ebdbd510dd6741832","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:30:32.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:30:32.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9ac0cee6dc9e41c0384048ce4f2ad4da83ae3f81c9ce1880a079d163b2bdb1f0","metric_details":{"best_score_across_scorers":"0.49","model_release_date":"2026-02-19","stderr":5.024183937956915},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4mzBGLow3HkS8BmAYCzGTR","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4mzBGLow3HkS8BmAYCzGTR.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4mzBGLow3HkS8BmAYCzGTR.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a4e2b2c26de40cd7af4bc10d2a3b9f5bc7b169791df428f06f162d984558a6a5","result":{"confidence_high":58.84740051839555,"confidence_low":39.15259948160445,"sample_count":100,"score":49.0,"score_display":"49.0","source_stated_rank":12},"run_fingerprint":"84fe919d41215707816c00c10e30d2f2f6e1d5787f2966b870e040e19641b33e","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":227,"at":"2026-02-19T18:01:28.796Z","basis":"evaluation_started_at","evaluated_at":"2026-02-19T18:01:28.796Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0151ef634ea2befb7aa3aa9b04a342e25eb644821188472cf82023eedd1c0c2a","metric_details":{"best_score_across_scorers":"0.55","model_release_date":"2026-02-19","stderr":5.0},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4QfM9qyeWGqNc8YTcQNV76","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4QfM9qyeWGqNc8YTcQNV76.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4QfM9qyeWGqNc8YTcQNV76.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7d1ec3054eb8cd156ed613b8ebb24182faff19cf3ecf6c0e9c440c5ebe50334a","result":{"confidence_high":64.80000000000001,"confidence_low":45.2,"sample_count":100,"score":55.00000000000001,"score_display":"55.0","source_stated_rank":7},"run_fingerprint":"b594504ec3159c8d00e125f19aabb9ba038fd91fd4e9658bc3269a9b0f17e569","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a10ffb1c1146967b80c00e2f","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ec355a8135b5dd8c7716bc126e2827d6e31f94d68e684d3a8b8f54114e0bfbb5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.5223","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Pro (Preview)","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"cc2abe7a87614a9ab147026b1341186e1362cd461e7cafc629916f4dc8517201","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":98.0,"score_display":"98.0","source_stated_rank":8},"run_fingerprint":"d54df226e0d3914be03d0892db3383f339b63c34430fb6da2a9a3a1841318493","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fbb279d340e3a9059784a4f6","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"88a2a13f3f3f9d44c92cbd5f2ef5caa5a91930a6dba4cbe076f6cf8f050467f0","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.783381,"date_is_proxy":true,"latency_seconds":312.261,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.83},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-pro-preview"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"0448a57885719a2cce30d9808455a2eee9aa490524faae010a4e720c251604a5","result":{"confidence_high":82.3868,"confidence_low":75.2132,"sample_count":500,"score":78.8,"score_display":"78.8","source_stated_rank":28},"run_fingerprint":"8564d86daaba2403a5ac38ca70a636bf78c72173315432ecf55e2ec5769724cf","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"f09c57a8a0af2f2cc0bbd086c5d7f44b8cf80d83d1d76613b8b6364cb86926a2","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-02-19","pass_at_5":66.36363636363637,"potential_contamination":false,"sem":1.199173268933902},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"Gemini 3.1 Pro Preview__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"264a1b30ff55d0e372595d202bedae349375f44074f5a61627eca3e3dd7d29e6","result":{"confidence_high":53.44128869801955,"confidence_low":48.74052948379865,"sample_count":null,"score":51.0909090909091,"score_display":"51.09","source_stated_rank":null},"run_fingerprint":"564af9ae9f84ba108ecbfca44c385f8016673f5283ddd1a26a20acf87b010202","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2f6483fa19b9e55f4f65fa80","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"24531e87c0072f7ff251aad47f74d4c8a8d1d9817f4fa9ec9a809bf051d0107b","metric_details":{"ci_attempted":452,"ci_half_points":1.481095461108845,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":53,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":75.56444444444445,"mean_cache_tokens":1678992.7311111111,"mean_cost_usd":2.1434045568888886,"mean_duration_seconds":867.7979107013274,"mean_input_tokens":2403677.331111111,"mean_output_tokens":28368.884444444444,"median_agent_steps":70.0,"median_cost_usd":1.7235064000000002,"median_duration_seconds":734.5636605,"median_input_tokens":1834116.0,"median_output_tokens":25990.5,"median_output_tokens_to_pass":23839.0,"median_peak_context_tokens":49745.5,"n_attempted":452,"n_passed":53,"n_tasks_attempted":113,"n_tasks_passed_any":32,"pass_at_4_points":28.31858407079646,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gemini_3_1_pro_preview_high","source_model_version":"gemini-3-1-pro-preview","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"2091ed8cc90c49695eccd5e5a9623bb42511a74e8bdc6eda0bbd3d1d5fcb0f69","result":{"confidence_high":13.206759177923006,"confidence_low":10.244568255705316,"sample_count":452,"score":11.725663716814159,"score_display":"11.7","source_stated_rank":68},"run_fingerprint":"856701cddec8a0a373d524fdfea4bece9abb2064d37b7c6cef5f69d0b5645322","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"68038cc412b492b8866b410e1c8e98b7f0cd5a8e343267f5615690e97545765c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"75.56444444444445","mean_cost_usd":"2.1434045568888886","mean_output_tokens":"28368.884444444444","model_release_date":"2026-02-19","notes":null,"pass_4":"0.2831858407079646","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-1-pro-preview (high)","source_effort":"high","source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"5d07eaa7feb74fdf8d787750e63661dda8c76aba2d2d3a1b387d2b77a27756de","result":{"confidence_high":13.206759177923004,"confidence_low":10.244568255705314,"sample_count":113,"score":11.725663716814159,"score_display":"11.7","source_stated_rank":67},"run_fingerprint":"8d26797d950770d5544812986f7abf02e6df53ee9963550370b25d9112cc147f","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_e361abe5266a3ddfdcef017b","split_or_window":"openai-provider-comparison","unit":"percent","version":"DeepSWE v1.1 · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"27311c264e8b48afc616eb9ff60fcb6b6743da4f40856e8fb4d8809f75ad2363","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · DeepSWE v1.1"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value for DeepSWE v1.1; competitor rows are secondary re-reports; exact scaffold, task coverage and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-19","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"coding-agent tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"98369008cd90a0aca4c2999ca1a3154c134d1cd628e75e9e8cc36313dcc6ae52","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":11.8,"score_display":"11.8","source_stated_rank":null},"run_fingerprint":"f157c854eb2512625a65873824a3b028cd68420c3e2b061280c9846a6601e85e","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · DeepSWE v1.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f89e750b6b0f35bc7646f10","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":179,"at":"2026-04-08T17:04:51Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-08T17:04:51Z","status":"stale"},"measurement_id":"cefeea68a38c7344cd51189e5ed07edc14bec5d88052dcd31da143ac63730541","metric_details":{"confidence_interval_half_width":3.6,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gemini-3.1-pro (thinking)*; createdAt=2026-04-08T17:04:51Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2026-04-08T17:04:51Z"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":true,"cost_cap":"uncapped","protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"gemini-3.1-pro (thinking)*","source_reasoning_label":"thinking-unspecified","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576","turn_limit":250},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"5c7a7e03cf04d095700bbb0f30dda302e0ba944247fbf29c239c63642b0ad6fe","result":{"confidence_high":49.7,"confidence_low":42.5,"sample_count":731,"score":46.1,"score_display":"46.1","source_stated_rank":5},"run_fingerprint":"5d81a10c0c13e0b52e4a0e01cd7594592653f67b9a0662c2491fd1b25af57442","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=gemini-3.1-pro (thinking)*; createdAt=2026-04-08T17:04:51Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0e8edd382c75ef7c82cf29df","provider_config":{"source_id":"scale-swe-bench-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_50545b611e994b43926cee37","split_or_window":"google-single-attempt-comparison","unit":"percent","version":"SWE-bench Pro · Google comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"0918eba981f9acaf103100597dd4c492e0991724fdda112e1085448a4e547313","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · SWE-Bench Pro (Public)"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google DeepMind provider comparison; single attempt; exact scaffold, task revision and thinking level not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Google comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"fcb7688c5ad7008bfff0beb916b43928af6d62ecd61c301f2be13832e9836ee0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.2,"score_display":"54.2","source_stated_rank":null},"run_fingerprint":"bdf30694998bbd1aa38e8891bd0146acf2ebe62205eabf24019113287ebc5568","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · SWE-Bench Pro (Public)","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_58b76c4ee41c5507274dd5cb","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_7664765571d731e0f6ff7adb","split_or_window":"openai-provider-comparison","unit":"percent","version":"SWE-bench Pro · OpenAI comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"c481756ea38910ad64ab4a8ff58c84214a62b84e136cd143a4647c69fe4e72c0","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · SWE-Bench Pro"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; OpenAI rows are provider results, while competitor rows are explicitly retained as secondary re-reports; exact scaffold, task revision and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-19","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"OpenAI comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"db1373bc2e64be3bbbca6e9454ab2071460c28f34a27308b1e6d9f5cb91d688d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.2,"score_display":"54.2","source_stated_rank":null},"run_fingerprint":"970b0c3fe4908af6864af120c0ac95d595ecb4ba898093a42ffb016f0500b2c6","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · SWE-Bench Pro","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f89e750b6b0f35bc7646f10","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":210,"at":"2026-03-09","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-09","status":"stale"},"measurement_id":"ae25c8eecfd663ac41637141b35664673eaad560476fdeab9935d3b65e044039","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":"2026-03-09","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"b135f4b467d586011ba96e921074bd22ba5666ca1223d94e18542068d02118ae","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":22.55,"score_display":"22.6","source_stated_rank":14},"run_fingerprint":"c8761a3c8c95f566a31ae74c20775157a24efa033f4e1e41e386b317dcc1cdde","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_73862c6e38001c572af5526a","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"054bc61d1722f0126a98dbbed999fd5dbc55ae8015d02d288eb75380d5ab9b56","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.25149,"date_is_proxy":true,"latency_seconds":722.637,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.589},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":65536,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"high","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-pro-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"27bed1c2a9b9c8d6ddd84b5116f96f465fc6d3adda0f0e2d8d78354a830f5d8f","result":{"confidence_high":56.90744,"confidence_low":46.758559999999996,"sample_count":null,"score":51.833,"score_display":"51.8","source_stated_rank":22},"run_fingerprint":"5585dcaee1895761e00ed115b72f7dafd70ba9a869331fe592c4333c69bc23cc","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3e966d145de69a2010577322f2a91c8921ed716cab8a1654db3f0013c2c223a8","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.101147,"date_is_proxy":true,"latency_seconds":88.891,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.931},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-pro-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"1343e5a060b95e52e9abf5b97386f54a3cd3881cbefe492d8e5abc1ed93f4d7d","result":{"confidence_high":90.30976,"confidence_low":86.66024,"sample_count":null,"score":88.485,"score_display":"88.5","source_stated_rank":6},"run_fingerprint":"68e31a87e5672ae289523436d0fbc3890334d8e4d3b59b0b851253726062d491","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f9d5590837947d135333c011018016e64f81fa015cb12e12ced2e4ec356b748e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Pro Preview","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e19ec468c2a5d484361cdde2c5ae2d160dfd77b7b5b9c540459be845a530fd7a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.912037037037,"score_display":"58.9","source_stated_rank":13},"run_fingerprint":"88b67801a25b7e5d9516eced92ff268506d0ee12f752dfb0320f441c3c8286b7","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b7b3ebda2c95fe4a3c175a73","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c757478187f55e8ae791026e3cb0151ae3fc26d42b901b774da19d23fc57f6ef","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-19","source_locator":"Leaderboard models table · Gemini 3.1 Pro Preview · scicode"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-1-pro-preview","upstream_model_label":"Gemini 3.1 Pro Preview"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"4b5eeeded06591aaf15fc1bc9ba4410b44ee5e6623a979de4aa73a030923f8b8","result":{"confidence_high":64.216905,"confidence_low":52.915677,"sample_count":288,"score":58.680556,"score_display":"58.7","source_stated_rank":null},"run_fingerprint":"9dcf593ed84c22f969b005fa8d6408a33b6afe44a1c29e942129ef8ce4ab430e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Pro Preview · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bf7a013a6ba02a14d73d95c","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e404df4b996ec3557d5f94587dab793c973287689ca3d20d3e4c2232add6b889","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.388403","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3.1-pro-preview-high","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"bc779e3326d0bf2798439e12ee5b09564e180b02c37dd637e5d73d8d1578620f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1160.6,"score_display":"1160.60","source_stated_rank":40},"run_fingerprint":"f0351487825bf44be233bb6ec6278144349fb5ba43b4645727e8c6bb9e12654a","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2b50318af890d1dc6735a84e425fbb9abc156ef6887473266497526dc984a807","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":3.825202,"date_is_proxy":true,"latency_seconds":1211.906,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.342},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-pro-preview"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"20f35d11bb3b54e58b49791e6a359541cea6293b1286369251631e3e50367801","result":{"confidence_high":40.54432,"confidence_low":23.52368,"sample_count":null,"score":32.034,"score_display":"32.0","source_stated_rank":65},"run_fingerprint":"10ecbb7c335501b8c796557f82777164f22368bf4c086a77c59c8a0baf409ce8","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"69bfa19f3fd41e3889d7d63750d37240ef3fca57d738ac2a319a89c4a293b7c3","metric_details":{"license":"Proprietary","variance":6.611030339333661},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3.1-pro-preview","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"71ae260b27376217b14f0c68515cd4a7cc2beae37263b9719243603fb4dd1b98","result":{"confidence_high":1451.239745328458,"confidence_low":1441.1608563399018,"sample_count":24347,"score":1446.2003008341799,"score_display":"1446","source_stated_rank":67},"run_fingerprint":"01775fda8f70d9d87384cbd896ea072087fe693abc7b2629c5d292485d5179e7","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f3c287b85bdf2bddbf2899eb","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aba848c1c906b4e8c724297192e93b62f120bcd8bdf1303a9bf43d0f81d0096e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"85eabe8dac94c2f69683ab6570fc0f057fa8cfedee8479546c0d2be5ada70aeb","result":{"confidence_high":1451.8,"confidence_low":1441.48,"sample_count":22891,"score":1446.64,"score_display":"1446.64","source_stated_rank":67},"run_fingerprint":"96dd93ada6973329fe43c138ce19e601a6e4ae438dd7e3a4d3e791c020831a75","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5a0c349b3688db3854729111","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"mirrorcode","metric":"mean_score","name":"MirrorCode","release_id":"release_07bf24088c0805f7b6df7a88","split_or_window":"15-programs-30-language-tasks","unit":"percent","version":"ML +Private 2L"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T16:12:46.343Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T16:12:46.343Z","first_observed_at":"2026-08-28T19:57:34Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e58de5624560412b4ace9cccd573d30f8e42e19286a4b5b6bbe0c1d64f38770d","metric_details":{"best_score_across_scorers":"0.08888888888888888","model_release_date":"2026-02-19","stderr":4.561939004190449},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mirrorcode-ml-2l-gemini-3-1-pro-preview","log_viewer":"","logs":""},"run_count":3,"scaffold":"Epoch AI Inspect ReAct agent","tools_allowed":"shell, text editor and visible-test evaluator; no internet"},"protocol_fingerprint":"4060ee56d737012bf0bdb88584e8d4acdfa2d0d138dc49898229b803c43191ea","result":{"confidence_high":17.830289337102165,"confidence_low":0.0,"sample_count":30,"score":8.888888888888888,"score_display":"8.9","source_stated_rank":9},"run_fingerprint":"ec7555dc825cdd2a21ba7009d827152a591b6818f094e6615c5db50caf2e363d","source":{"content_hash":"3e212e97e03d0343e676ecc7ff046ee7b94d219ba7d863364634b7361dc89b4b","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://epoch.ai/benchmarks/mirrorcode","id":"epoch-mirrorcode","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · MirrorCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/mirrorcode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dd279f0934d3fbcc8d8df5b8c0ea370e07999b5952529cdd5023f7b2cc55d1fc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"2d55e99abd048ba98d8233abd81925bb13dcd26af11a5c45a211addf5d7e63d1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.07,"score_display":"72.1","source_stated_rank":31},"run_fingerprint":"588cb8492f0767ca66c8d1f8cd075d85baf1debaa62b407546d9677fbccf1a2d","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_32dba87df2012b02f4613fb6","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":217,"at":"2026-03-02","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-02","status":"stale"},"measurement_id":"455aab1e0d87af817531a02c95d07cdaa9b9fa12c1801c25afe66317a6ca6b7b","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"standard_error_points":1.7999999999999998,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":"2026-03-02","source_scaffold":"Forge Code","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Forge Code","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"56b05aa55fa5919e39b4998d2b985e6f0e0d3d4de0b653e59c74aa265b27a38c","result":{"confidence_high":81.92800000000001,"confidence_low":74.872,"sample_count":89,"score":78.4,"score_display":"78.4","source_stated_rank":10},"run_fingerprint":"6762dc99d31b0fae6af17e726699b9678ab85a21ef72f5794c87f222fbdb6588","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_496cb04156bf8959054e7bf6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":217,"at":"2026-03-02","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-02","status":"stale"},"measurement_id":"e52606ab3493a99c8c829dfa5195a6a0d38b016c0efc5a37e378284df1943848","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"standard_error_points":1.8000000000000003,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3.1 Pro","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":"2026-03-02","source_scaffold":"ForgeCode","upstream_leaderboard_url":null},"run_count":null,"scaffold":"ForgeCode","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"9204b99ed566d87820b160bd7a0cd15c12936cb6c7a0b875a93721228739b4fd","result":{"confidence_high":81.92800000000001,"confidence_low":74.872,"sample_count":89,"score":78.4,"score_display":"78.4","source_stated_rank":10},"run_fingerprint":"2633e993d52aaa8e1f95219d542909cfb6401830f93145393c2653a5b7a3aafa","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_496cb04156bf8959054e7bf6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":228,"at":"2026-02-19","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-19","status":"stale"},"measurement_id":"e2243d01241ad3bb4c5a37e56a95b8879f72f8f4042b5cd04b1c9becfb048ccf","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"standard_error_points":2.6242804005,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3.1 Pro","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":"2026-02-19","source_scaffold":"TongAgents","upstream_leaderboard_url":null},"run_count":null,"scaffold":"TongAgents","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"f7466c7a8903650cb2691adba3e31a1bd5797b8e82c1b597c0330294e5160f26","result":{"confidence_high":85.36830868608001,"confidence_low":75.08112951612,"sample_count":89,"score":80.2247191011,"score_display":"80.2","source_stated_rank":6},"run_fingerprint":"0f914fd705bc1ea3fbf95d919056ec1cc9abb35d990ca9373247192f2eaf0934","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_496cb04156bf8959054e7bf6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":228,"at":"2026-02-19","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-19","status":"stale"},"measurement_id":"9fe026ef2ca2e3d0ea237e0e04bcc1962ea161682f38ec5746bfd9460330b2f3","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"standard_error_points":2.5682394863000004,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3.1 Pro","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":"2026-02-19","source_scaffold":"Terminus-KIRA","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus-KIRA","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"25c99ea3644274dc5449dd97cb6b04ad7076686080e4555775d9be2db9f6d614","result":{"confidence_high":79.865210067348,"confidence_low":69.797711281052,"sample_count":89,"score":74.8314606742,"score_display":"74.8","source_stated_rank":18},"run_fingerprint":"8096bfc151bd90b7bb2043fa8c40f163a7fc90a9585c21db957d098b638937ee","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_496cb04156bf8959054e7bf6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":228,"at":"2026-02-19","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-19","status":"stale"},"measurement_id":"6792a0034c5eeebc646ea11f0cdd0b4e4c4851de89752d5218e40baa8ebefff3","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"standard_error_points":4.2009785316,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3.1 Pro","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":"2026-02-19","source_scaffold":"Gemini CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Gemini CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"ed2ab05c9df42a67ccd94fd3fb5fcb8fb5f5c0729e7d5d11f45160ca884a2b9f","result":{"confidence_high":67.597213802036,"confidence_low":51.129377958164,"sample_count":89,"score":59.3632958801,"score_display":"59.4","source_stated_rank":50},"run_fingerprint":"9dd3336a43631d6323b44ec3d59e4d42456930813ce9b32735d3bf62a1a46539","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_496cb04156bf8959054e7bf6","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"902885c40fae5e6eb6da08d9f6f2d922c5d8b172cdefed43b30d265c2419771c","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.582677,"date_is_proxy":true,"latency_seconds":410.249,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.124},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-pro-preview"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"354a54a08b6a7a95d39cf944b756511b4663a3d3604560c71b227f3a289c1b91","result":{"confidence_high":72.99004000000001,"confidence_low":68.58396,"sample_count":null,"score":70.787,"score_display":"70.8","source_stated_rank":27},"run_fingerprint":"601a85fbbcdc54a2bfa114bde268d57f8713bf783996d198313e98ad7a7a6aa1","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"22d117bd99ed48eb8e51a182eb6e7d2e4452e6822729d4679aaefc92e2bc228a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.1 Pro Preview · terminalbenchV21"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-1-pro-preview","upstream_model_label":"Gemini 3.1 Pro Preview"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"80f5f07a12f3e2f3a9aacb7a3ad0fe98ca6927de8468f2a3e98d05ab7b848158","result":{"confidence_high":81.799199,"confidence_low":63.798178,"sample_count":89,"score":73.782772,"score_display":"73.8","source_stated_rank":null},"run_fingerprint":"43a84826d34050b9ec685a568316c4466769a8300848b01ac738de81989041fb","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Pro Preview · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bf7a013a6ba02a14d73d95c","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_0f4d0bfc0ac73283ea3a731e","split_or_window":"provider-comparison","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":76,"at":"2026-07-21","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:32Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-21","status":"fresh"},"measurement_id":"4ee0dc5860651f64483569205766f04b676fadede2b5642cbf3593d027db20b1","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/","evidence_sha256":"d148585d223f2bf4596bd5dca47ca518be4891525cd3e18a3eeb7599455505c4","kind":"reviewed-static-report","published_at":"2026-07-21","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/","version_id":"e2b9f9e2736a881433c558e4098f782716af51b1e54bb054f3f055aac8b3d45a"},"source_locator":"Model-card benchmark table · Terminal-Bench 2.1 comparison"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"current Google provider re-report; effort not restated","reasoning_effort":"reported","result_published_at":"2026-07-21","results_as_of":"2026-07","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"aa275c6d076871ccb0440ce95de82b94963f9d951e6a1772b068d42ff28607fb","source_published_at":"2026-07-21","tools":"terminal agent toolset"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"3d44058f61b6ff248c81adb739cba2b8212fcf8445873a4b85671a296a1a4b14","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.8,"score_display":"73.8","source_stated_rank":null},"run_fingerprint":"59a505140e233e68dc86477b8f2a8565499841c37c9b7441b48a8b29a46df3b9","source":{"content_hash":"00b894325b1d599b95a6f00fa33353412d37818fa36735786b1cc1e024427117","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/","id":"google-gemini-3-6-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · Terminal-Bench 2.1 comparison","name":"Google DeepMind · Gemini 3.6 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_15d0b6d60acdd0a2cab3c34a","provider_config":{"source_id":"google-gemini-3-6-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":4,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"secondary-re-report","verified_status":"secondary-re-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_39c649809acb869ddd02ba8c","split_or_window":"terminus-2-provider-comparison","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"39f85a3a0861c4710a352cbdfcf768258e57b8fd0b7967dfc12e0b66ef0ed4d3","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · Terminal-Bench 2.1"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider comparison; pass@1; thinking level not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"572100f7f4354d5d61b8bcb74e39452e5e90a94ab13e13f8f5a7c5249759e35b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.3,"score_display":"70.3","source_stated_rank":null},"run_fingerprint":"d3c8d595ed8aaa79a1f6d43a8c8e55f0944efec9b45891e7ece36567b5df823f","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · Terminal-Bench 2.1","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_58b76c4ee41c5507274dd5cb","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_6d7e85c927ba2919129973e2","split_or_window":"openai-provider-comparison","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:30Z","last_confirmed_at":"2026-09-05T20:09:26Z","observed_at":"2026-09-05T20:09:26Z","source_freshness":"expired","source_observation_age_hours":714.27,"source_observation_status":"overdue","source_published_at":"2026-07-09","status":"stale"},"measurement_id":"221679fc1a72e332632e1267968c41d082a261e5044208e89aa7f4b63b4ab949","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://openai.com/index/gpt-5-6/","evidence_sha256":"625395d5375b470864f215e82049987cdd50aef9f82812a7de29ed77889a7167","kind":"reviewed-static-report","published_at":"2026-07-09","reviewed_at":"2026-09-03","source_url":"https://openai.com/index/gpt-5-6/","version_id":"c13d236cd914d4cdddf71994d89c63d98c6cb405dfcca29adc44947db39d411c"},"source_locator":"Benchmark table · Coding · Terminal-Bench 2.1"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"OpenAI","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenAI launch-table value; competitor rows are secondary re-reports; exact harness and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-19","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"0aad52da34c26ee5b0b2b7489177a16fd90b1bd225efe15d0ac534d38aa34a8d","source_published_at":"2026-07-09","tools":"terminal agent toolset"},"run_count":null,"scaffold":"comparison harness not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"6a03fa8a13a2239359cbd63cf4718422cb792ffc7263a134d6ba4835e7586d5a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.7,"score_display":"70.7","source_stated_rank":null},"run_fingerprint":"e36dd6fbd3c5eca1d70e26202939b538b6ded53f58ea1ec8f75850ef00360f5d","source":{"content_hash":"4da616b59d68d3a7539bb4fe51571e92db075cb2938198f21a3d4edb838d1081","fetched_at":"2026-09-05T20:09:26Z","first_fetched_at":"2026-09-05T16:49:10Z","homepage_url":"https://openai.com/index/gpt-5-6/","id":"openai-gpt-5-6-report","last_fetched_at":"2026-09-05T20:09:26Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Benchmark table · Coding · Terminal-Bench 2.1","name":"OpenAI · GPT-5.6 launch evaluations","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://openai.com/index/gpt-5-6/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f89e750b6b0f35bc7646f10","provider_config":{"source_id":"openai-gpt-5-6-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":153,"at":"2026-05-05","basis":"evaluation_at","evaluated_at":"2026-05-05","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"002ba31088c7e8a0e6e616ea25c72ed071f61d7fde487a225e09021da90dd266","metric_details":{"cost":null,"pass_2":15.12,"pass_3":11.08,"pass_4":9.28},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"gemini-3-1-pro-preview_sierra_2026-05-05","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"cc2b5f2e843b4d0088ddb39baa3601946a893dd4972691552d040c8ddbfcbdb9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":26.03,"score_display":"26.0","source_stated_rank":null},"run_fingerprint":"696a37daac5744daf9cd28804f874b76513e6dd8f6d0e906fc7473b1ad8a0d19","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b11a4c7727d36497db90df490e6c77a458508b7516c5ac34883014b1bc3b6239","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-19","source_locator":"Leaderboard models table · Gemini 3.1 Pro Preview · tauBanking"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-1-pro-preview","upstream_model_label":"Gemini 3.1 Pro Preview"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"aca520cd4821828a901ec2b7209a2bb93dc4b6fce41b6fdd12aefccf6205875a","result":{"confidence_high":30.615467,"confidence_low":14.446889,"sample_count":97,"score":21.443299,"score_display":"21.4","source_stated_rank":null},"run_fingerprint":"14d6cd1afa5123a89019541f347af0077acced73734e950408eda0c00c7fa341","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Pro Preview · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bf7a013a6ba02a14d73d95c","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":314,"at":"2025-11-24T18:59:59Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24T18:59:59Z","status":"stale"},"measurement_id":"b0fbaab94ac3558b6b753e8e07c53b0e36f3d17837d5c61af7e686dd760ef2a4","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.5,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gemini-3.1-pro-preview (high); createdAt=2025-11-24T18:59:59Z","source_row_created_at":"2025-11-24T18:59:59Z","task_pass_coverage_threshold_percent":75},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"gemini-3.1-pro-preview (high)","source_reasoning_label":"high","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"786f826609cbc2f7072b1ea3d499bc14efebad9bcd67198ad559c5a03a9d33c5","result":{"confidence_high":80.7,"confidence_low":75.7,"sample_count":1000,"score":78.2,"score_display":"78.2","source_stated_rank":3},"run_fingerprint":"18ab0cc65c7afb2722aec2a15571026e7848c97c2b8370c42552445ac7442a75","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=gemini-3.1-pro-preview (high); createdAt=2025-11-24T18:59:59Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_dd3be8508497fcbd983b0185","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"b8796af64635661398b904d63b66723dc1024b861c50a18b65ac29e8056a40a5","metric_details":{"expected_trials_per_mode":261,"normalized_gain":38.7,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":24.8},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"gemini-cli","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"gemini-cli","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"b364c19f09c3235a2849dcf46f5f7497a801c57e4a3db1ecf5efd428de0c8418","result":{"confidence_high":70.1,"confidence_low":51.5,"sample_count":87,"score":60.8,"score_display":"60.8","source_stated_rank":4},"run_fingerprint":"cd788dad355a4a048591bb5e84b111298c20de68aefde7d8dae02cb0ae260e9c","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_522565efd30b538ee4ac5d42","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"5b2eb252e580d4084def0edb81ad9c582b8933cfe85133b29bb5ad580f49de5d","metric_details":{"expected_trials_per_mode":261,"normalized_gain":28.7,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":19},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":62.0,"confidence_low":43.599999999999994,"sample_count":87,"score":52.8,"score_display":"52.8","source_stated_rank":11},"run_fingerprint":"6668fc5059c005e185698057e55fad31dbee3714d157bbc462fe7b89f375bae9","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_522565efd30b538ee4ac5d42","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"2ae918cc858ec8eb29707b7b1ff2168a45d43b8d20aad269288b4c4a62d4bfe3","metric_details":{"expected_trials_per_mode":261,"normalized_gain":38.7,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":24.8},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"gemini-cli","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"gemini-cli","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"8c3496e82aac91ec0feee659c4f06e8ed97c98e977dbb8aefe3da75bd141f00a","result":{"confidence_high":45.6,"confidence_low":26.4,"sample_count":87,"score":36.0,"score_display":"36.0","source_stated_rank":7},"run_fingerprint":"6c1f9d79993fd2f1d046eab7669320e4124766c72229dddda884815cb84d6f23","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_522565efd30b538ee4ac5d42","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"aba76b3860a7a150b6472379ee405170343d653f638f244bfcc4e931ce7ef6a0","metric_details":{"expected_trials_per_mode":261,"normalized_gain":28.7,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":19},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":42.4,"confidence_low":25.199999999999996,"sample_count":87,"score":33.8,"score_display":"33.8","source_stated_rank":9},"run_fingerprint":"9e230a6131115332aef6b93b1943312dbd67bc4831bdd66ae456a976e8697f41","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_522565efd30b538ee4ac5d42","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"409c98f52a105b532e77d265a2d3802611bf92010d4d0e3729f3c9790c092102","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.525","mean_standard_error":"4.6","model_release_date":"2026-02-19","notes":null,"standard_error_points":500.0,"upstream_source_url":null},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Pro","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"d6c4d56064debae59c97d146059acbc9973211d9076728b934419dc69c44becc","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":35.3,"score_display":"35.3","source_stated_rank":36},"run_fingerprint":"bcf9fe1b84def3126c3e805d63f6197a2d4af696f0f8b824272663cde630aa82","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3ad665509e47df167de108cf","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ac5304cfd9b2872cf79573b275d5d00f6360c3a04b95f60d68e0a8265d113c26","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":86483},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Gemini 3.1 Pro Preview","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"73ceff85d359e948d47ec21de7d94ad89a9a7f0da64623773c861fc56952d3c3","result":{"confidence_high":-0.06732810566074737,"confidence_low":-0.08690533911564136,"sample_count":3446977,"score":-0.07711672238819436,"score_display":"-0.0771","source_stated_rank":46},"run_fingerprint":"e2d7573ccac984e890d7e367960e6d926bee0df0f7daf6430506f404f37a82e9","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_71d78988738e81ae5ae5bd43","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"98e67e6e1a307972765e7f3a42541b2c8fe4c489a5b1d700501d66ec1ec8efdc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Pro","source_effort":null,"source_model_version":"gemini-3.1-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"8259756d9448652adb9c8e2ee1fe06f67271648e2f5f364b0f1709d484d02ab4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":911.2139999999997,"score_display":"911.21","source_stated_rank":51},"run_fingerprint":"90f83ebf338a19e3b45704d01bd970c4a75d315c435ef4d004ea05d41f4a7df8","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e91ccba2908d54447036bd3b","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"583836730243d270a09f194124893687b594fdc146cda0821224df58d1cbbd0f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-19","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Gemini 3.1 Pro Preview · gdpval"},"model":{"id":"google/gemini-3-1-pro","name":"Gemini 3.1 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-1-pro-preview","upstream_model_label":"Gemini 3.1 Pro Preview"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"f9b394be027eb1e756a39204d1b5a5c607843c200c8b56ff129c70fc0ab5a4f7","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":775.98,"score_display":"776","source_stated_rank":null},"run_fingerprint":"5b58539fe601bcca30fb5b4d0dc6ba9b68d1355a8ddb8204880efb28ef08e9ae","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Pro Preview · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bf7a013a6ba02a14d73d95c","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c78caa53cef0b062b0f52126008ab4bea02e1b4c4f06f9430768e99b64284603","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-31"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: DeepSeek V4 Flash 0731; model release: 2026-07-31","source_accessibility":"Open weights (unrestricted)","source_model_name":"DeepSeek V4 Flash 0731","source_model_release_date":"2026-07-31","source_model_versions":["deepseek-v4-flash-0731_max","deepseek-v4-flash-0731_unknown"],"source_reasoning_efforts":["max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4d4cd52852efa1d7b364ad408b18bcba1332af748ec8a4e5911cd79a36d29c1f","result":{"confidence_high":156.56,"confidence_low":152.02,"sample_count":null,"score":154.53,"score_display":"154.53","source_stated_rank":32},"run_fingerprint":"ff03e5086202da069689c209877d6c2eb44e94daa3dcaba6b60c3929bbc0af45","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_21b864923f5257b0d3f99f91","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5f3dec248afdb6c329d0cc474682a44334e5906a8519974e80d59d033135a5f3","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-31","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · intelligenceIndex"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash","upstream_model_label":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"ec4ca39030893f1a4173036cadddd71bf7101e34a8da102eca8820e654db9e9c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.528188,"score_display":"34.5","source_stated_rank":null},"run_fingerprint":"3c5738d2e3a7a2080d82e9c3788d9db259762876ef3d56dc9d2e4f021fe8bcdb","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"89eeb9283f2fd2771ad349afd4702b49abf57feb29b768143b77cc2a6b1a029e","metric_details":{"category_scores":{"Agentic Coding":46.76766666666666,"Coding":74.9845,"Data Analysis":79.32733333333333,"IF":65.51675,"Language":79.175,"Mathematics":86.78975,"Reasoning":86.6345}},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.17078571428571,"score_display":"74.17","source_stated_rank":42},"run_fingerprint":"886b8ca4fb6a60ba4dc708c82ba976dcb5d49f629710a5b360f4c3589bc7a615","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_27322121cfe7d870e70b359b","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_1f4bfa5486eb4deec4942449","split_or_window":"provider-report-no-tools","unit":"percent","version":"HLE original"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":66,"at":"2026-07-31","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"fresh"},"measurement_id":"ad5a5f2007ee39c3de3855146704d005e804b1cd8a340ae8af18b48ea7e082d2","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table · HLE (wo / w tools)"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"DeepSeek model-card HLE comparison; tool status reported; exact dataset revision and judge not restated","reasoning_effort":"reported","result_published_at":"2026-07-31","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4589a852099f2d691fcd122c2a19219befde1963631538e6595ceb9eab750ee9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.8,"score_display":"37.8","source_stated_rank":null},"run_fingerprint":"b35e205fb2026d0fe75b0db6e96838ec878dbadaecd2711a24b816baae632fce","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table · HLE (wo / w tools)","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bf407507bedf02addf42d3d4","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":66,"at":"2026-07-31","basis":"evaluation_at","evaluated_at":"2026-07-31","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8904968cbf812eadcf0216c98fe4784dde142ba50b829336fe4746d9e97e7945","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/deepseek-v4-flash-0731","source_model":"DeepSeek-V4-Flash-0731"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4f5279aa3103c930d2ca025c426756999e179d4bb4596828c30d2ae918ace208","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.8,"score_display":"37.8","source_stated_rank":16},"run_fingerprint":"bbeef42204a5fd4b7dab8078165f578f48397931f623641ba06b7ea730b79ab2","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_40ecda814c8b3959b603d727","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5dc5214f4902e17be96d77ff","split_or_window":"provider-report-with-tools","unit":"percent","version":"HLE original"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":66,"at":"2026-07-31","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"fresh"},"measurement_id":"53cc1f69ab2b4506709d4478f876b4e94be6f4fbceb16baaffca881296dd1ad9","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table · HLE (wo / w tools)"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"DeepSeek model-card HLE comparison; tool status reported; exact dataset revision and judge not restated","reasoning_effort":"reported","result_published_at":"2026-07-31","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"tools enabled (unspecified)"},"run_count":null,"scaffold":null,"tools_allowed":"tools enabled (unspecified)"},"protocol_fingerprint":"5dae25c816094ad7f8f26486cc3fb3c5350d3d494ec7947796283e7547a1a13c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.5,"score_display":"51.5","source_stated_rank":null},"run_fingerprint":"434785edbbec5141793d0799cde94232915d85999d976f615c6fbb7945a7aa7b","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table · HLE (wo / w tools)","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bf407507bedf02addf42d3d4","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eda9f18914d5a10c5464700cad7866509e66c065784f9e6ae59200b3dff9c3fd","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-31","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · hle"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash","upstream_model_label":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"db5b4c9bc368ca3269fc65cd7577a445e70e6dca8d81c2798448ac5be6d6d1a3","result":{"confidence_high":40.626415,"confidence_low":36.522697,"sample_count":2158,"score":38.554217,"score_display":"38.6","source_stated_rank":null},"run_fingerprint":"d0024344ce46cac3c345b8e2c5d95f2754da8a6028258cc1ca400b32dd6e1b7e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:30:16.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:30:16.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dac6c5a727306a91838dbfcdb5888305b77f92f5cdb51b60c489a6a021bf046b","metric_details":{"best_score_across_scorers":"0.33633633633633636","model_release_date":"2026-07-31","stderr":1.495531772489948},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"TqFmiGvqozt7TSZYmhgF8m","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTqFmiGvqozt7TSZYmhgF8m.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/TqFmiGvqozt7TSZYmhgF8m.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"c508a328505e9ef3dc1b69c8bfd1933ded821747cd869563855260a91233d886","result":{"confidence_high":36.56487590771393,"confidence_low":30.70239135955334,"sample_count":1000,"score":33.633633633633636,"score_display":"33.6","source_stated_rank":57},"run_fingerprint":"5c2bff3915c226040f295dcf4083638e81c64c3e81690be5729948bb3a131d7e","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"87c4f1ce6b56b81d2a0033dd17db3985007b1b13a119c35c254924fa1c810c9a","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.012875,"date_is_proxy":true,"latency_seconds":79.956,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.694},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":89.899,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":256000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-flash-0731","zero_shot_accuracy":89.899},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"c8e4b8fa95f1f169dfcaf037f54373bbc25a723aa6ff66ce8e5cd23f88686400","result":{"confidence_high":93.21924,"confidence_low":86.57876,"sample_count":396,"score":89.899,"score_display":"89.90","source_stated_rank":27},"run_fingerprint":"ff2f375c25fa997e09c2b8ab5ac6c7197f722eac6c6b141f4a4b738caaa00994","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"def6ce9237f3d501a58ddcc56e5958c7d9b0bc7a6c55a9831b2ce6455072a51c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-31","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · gpqa"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash","upstream_model_label":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"78da0ebcf47fba0aecec48249a0b8de9a264ef05e95d10812ea8625c50194310","result":{"confidence_high":94.092166,"confidence_low":85.970617,"sample_count":198,"score":90.808081,"score_display":"90.8","source_stated_rank":null},"run_fingerprint":"9b97b4aa056cda22e63ea99997a4c78aee1f0b1410bccd59f5a5e8fe83d81045","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":64,"at":"2026-08-02T00:22:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T00:22:31.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"526b3c3ac4de400051dd06fecdc8bba50115f97e91e23bfde7327e6a58657e29","metric_details":{"best_score_across_scorers":"0.9103535353535354","model_release_date":"2026-07-31","stderr":1.7655466909172535},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"EDFusDaaFkh5n2MqKNxMBD","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"89a843f60d34cdb76254a6543b50389f28776d3e3f421336fe8734f471ef4eed","result":{"confidence_high":94.49582504955136,"confidence_low":87.57488202115572,"sample_count":null,"score":91.03535353535354,"score_display":"91.0","source_stated_rank":32},"run_fingerprint":"3f618636100859967525c2ce3218934788c98a022ac2526d1b201d0c04556df3","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"be00716b09ebd408cef3568cd68ea2111134faf7bb4a9f38d6a45465fe26f604","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.00265,"date_is_proxy":true,"latency_seconds":17.188,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.339},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":256000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-flash-0731"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"628de50b24afec62b903d08b85c9939b1a8a27243c229994f07708a6eb710032","result":{"confidence_high":86.87044,"confidence_low":85.54156,"sample_count":null,"score":86.206,"score_display":"86.2","source_stated_rank":47},"run_fingerprint":"e22f546bfbeecbafd017a313317f1bb0c6a6b1b072aa86e717530a633b9c8d55","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T00:22:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T00:22:31.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"487f624ec659587a3e915d6d8791cb483083f316f892ad9bf5be163593134d62","metric_details":{"best_score_across_scorers":"0.9444444444444444","model_release_date":"2026-07-31","stderr":2.0854366486931895},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FQ3Suh2vKacz7UNqJybTWk","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"48339610facb782b0d911290d11ed2dbf3217eae2adf9f0f02530b418cdbab4f","result":{"confidence_high":98.53190027588309,"confidence_low":90.3569886130058,"sample_count":45,"score":94.44444444444444,"score_display":"94.4","source_stated_rank":48},"run_fingerprint":"24dccc266861cebab57e48841f9ec9033d2bf07c0ff1c3508a868a2e15e4d6ac","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T02:15:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T02:15:37.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ae4a5d6515ca29978ac29d17697e21a810e45a468b38bd0608a09e7b7c3bbc85","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5754385964912281","leaderboard_private_problem_count":285,"model_release_date":"2026-07-31","public_example_count":10,"stderr":2.9329899789943816},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"mKhVMEicZeGjdvJdTRd7mK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmKhVMEicZeGjdvJdTRd7mK.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/mKhVMEicZeGjdvJdTRd7mK.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"DeepSeek V4 Flash 0731 (max)","thinking":null,"upstream_model_id":"deepseek-v4-flash-0731_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"f2ceab1ba37fc2104982d325b74da10d78a9ead35ee022f27dc7cc583476e02f","result":{"confidence_high":63.292520007951794,"confidence_low":51.79519929029382,"sample_count":285,"score":57.54385964912281,"score_display":"57.5","source_stated_rank":38},"run_fingerprint":"99b3d88714952effe21128e8356d4a6b3c585a89c1d08ff10b716201f4f03a4a","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aa382b8129f4ca5b1437b8fbf30fcbd397e027987342c8e42148c5e3c318016a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.04163034416666667,"model_release_date":"2026-07-31T00:00:00.000Z","results_url":"https://arcprize.org/results/deepseek-v4-flash-0731"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"deepseek-v4-flash-0731","model_type":"CoT","upstream_model_id":"deepseek-v4-flash-0731-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ed6c05b974d18e2d530771f8572131d611f108e3703feaf748158ccf388ca042","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.38888888888888,"score_display":"61","source_stated_rank":72},"run_fingerprint":"b58e6b85e6d68b0c1ad9f4b8add3f6b015ff7ee2abaadd423a14490295e84261","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f1a0a0fe5662a17e2f7c72b12e0cce54dc8092a90456e2294f5362ac1955fbb2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0453381305,"model_release_date":"2026-07-31T00:00:00.000Z","results_url":"https://arcprize.org/results/deepseek-v4-flash-0731"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"deepseek-v4-flash-0731","model_type":"CoT","upstream_model_id":"deepseek-v4-flash-0731-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ce0d1797311b978c4f043de1b2a11336179a55a593dffa015ef37d17ba85d635","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.972222222222214,"score_display":"56","source_stated_rank":86},"run_fingerprint":"a540a591e56646f22941ec6735181a22a577a3d2e3b452d855da796d1152830e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"df689a384cd85456db2e77533407d78b4fba0754cbec04aaec3f6191234a4c1b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0206284715,"model_release_date":"2026-07-31T00:00:00.000Z","results_url":"https://arcprize.org/results/deepseek-v4-flash-0731"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"deepseek-v4-flash-0731","model_type":"CoT","upstream_model_id":"deepseek-v4-flash-0731-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9b3c10a72b53f1596fc947a65e111fb833cf6ce78dcce9992ca0412bd8fab77c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.97222222222222,"score_display":"46","source_stated_rank":98},"run_fingerprint":"51dde968c8d581cebcf669ee9c8bb5e949ff16ae0dd372914e3cb76bc4ef535f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_dd089538c848df7e5e4d1310","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"faadd657519cdc7b3533f3d11a9db0ec509ddbf39c852b84fbf18d64c34feec7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0037609368333333336,"model_release_date":"2026-07-31T00:00:00.000Z","results_url":"https://arcprize.org/results/deepseek-v4-flash-0731"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"deepseek-v4-flash-0731","model_type":"CoT","upstream_model_id":"deepseek-v4-flash-0731-none"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d88e9420a6645f7b56295bd19c14794cfb0c33f4d6b731c2cd00b5f0d3ddb979","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2.083333333333333,"score_display":"2","source_stated_rank":194},"run_fingerprint":"11de23b6ba8147a306c15577472a1064113ea6a436ba0147faa4ee5c03968ae7","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e29d4b74c396a06029c21050","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"115bd05bd4d6f54607d536ba42ac17b58b8ceb798c90d8892f4d5a77aa52c48f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (Max)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"142511a92e973ed2519b5ce6e0d61ad001ae328b9351023c0b93ba76a8829469","result":{"confidence_high":66.272744,"confidence_low":56.264536,"sample_count":360,"score":61.38888888888888,"score_display":"61.4","source_stated_rank":75},"run_fingerprint":"946b645095774fe400af6d5cbc87644f1d34a7bd6a67dce0f7bd91d4f8582bd5","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"babc4211b0f1281912dfa7e52edbcdc334961a5c9c1360aa87b1033d9827d691","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (High)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b084b0f80301b4cba82263ffcc08601831c46f82ddda57b64b2d04ad2a4ef18c","result":{"confidence_high":61.010487,"confidence_low":50.807843,"sample_count":360,"score":55.972222222222214,"score_display":"56.0","source_stated_rank":88},"run_fingerprint":"ebd113bfd230636bbc70efbc5d7afa8b074337a4a496b3b5c8676552a45f1a53","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"921e32d3010f789704862567a1828ddc48a822209f9635b45dcf49f822aedde0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (Low)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ac65f95fb45b942af3bb0dc9c0f92aa2b8f9e90632eece40699dd29f89151fc5","result":{"confidence_high":51.135943,"confidence_low":40.893556,"sample_count":360,"score":45.97222222222222,"score_display":"46.0","source_stated_rank":101},"run_fingerprint":"8cb6ed43b4bd00fdcc53f759da6a233279ffcb3ee4e7e54de7a9f78e9ab2e0c8","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_dd089538c848df7e5e4d1310","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1b3a6f578138a4f3a58c9df9c4fd7d9e12926ce4c3e09f64fad82f5246292ddf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (None)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d70d36a66edf93732a5ec912508a446b4ae82bbc83b8afe8d15e2f7286f038e0","result":{"confidence_high":4.141615,"confidence_low":1.036903,"sample_count":360,"score":2.083333333333333,"score_display":"2.1","source_stated_rank":188},"run_fingerprint":"3b437021540d2ca2b9b01a5df95503e10f960ffe146aa47068ee321fae40b434","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e29d4b74c396a06029c21050","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T02:15:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T02:15:37.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"80c112b84f9d1f145b10d552b93f0a83b171fb5fff9545ce2e6097776c14a2a3","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.24390243902439024","leaderboard_private_problem_count":41,"model_release_date":"2026-07-31","public_example_count":2,"stderr":6.789956540036613},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"WYs83xB6NwrkMNyQJtuMcQ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWYs83xB6NwrkMNyQJtuMcQ.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/WYs83xB6NwrkMNyQJtuMcQ.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"DeepSeek V4 Flash 0731 (max)","thinking":null,"upstream_model_id":"deepseek-v4-flash-0731_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"b6075d68f0e8d230cdba359167d52f612e596f30c0039c7dab990949d63c86e8","result":{"confidence_high":37.69855872091078,"confidence_low":11.081929083967264,"sample_count":41,"score":24.390243902439025,"score_display":"24.4","source_stated_rank":42},"run_fingerprint":"550160fe53382b4f3edfa04e33ab9e55112384b57f1cca387a0b5f553bf47714","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2c5d70c2514d9d5e2afce08f3d60285518b15fff42db1728b05c14125b358a28","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"13e703b91dea95a15b992307a3254de239188ff3f6b755b1fb90218182d49e2c","result":{"confidence_high":26.884354,"confidence_low":9.690263,"sample_count":71,"score":16.5714285714286,"score_display":"16.6","source_stated_rank":72},"run_fingerprint":"624f1928334f1f1f67c85faff3f7073b8744db54be76578cf707c007015c82b5","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5cf6a3e442492c984daa413449268f0b2661cbd527be273ba1344e3ad1aa1d61","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-31","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · critpt"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash","upstream_model_label":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"71c1ef517fae994939ed6031a7be314280d251cf3237e2d39146cf104a88969a","result":{"confidence_high":26.967943,"confidence_low":9.65315,"sample_count":70,"score":16.571429,"score_display":"16.6","source_stated_rank":null},"run_fingerprint":"df7bd7ec1d910c0278d434bd48d907eac32bd852087f6b1a77e0cc0312307cc0","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d21f8ca3e52288c3446578dbff4ec37a3c3b6a99c8c512b2a9ca70029d8eced4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-flash-0731_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"cf7f78b05898922292516882047806d7f9daa066f7ec01a61f921883183837e4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.1,"score_display":"61.1","source_stated_rank":27},"run_fingerprint":"f6661cbbe24c63cc92920bcd19fdd137237ed98d2752a7edb1eb602edae26d24","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e2ef717a623eeb73e7398005","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T00:22:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T00:22:31.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"15a6072346d343c7abdced1855200dff855c6bb2adb7d16f0933907002a53f85","metric_details":{"best_score_across_scorers":"0.33","model_release_date":"2026-07-31","stderr":4.725815626252608},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Kk4oLoTLyDuqfnbqjPSvgZ","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d750208d0ff5b6d8c1283d596f93674c84b4bca8ad998b3c74b8ee50d869bf86","result":{"confidence_high":42.26259862745511,"confidence_low":23.737401372544888,"sample_count":100,"score":33.0,"score_display":"33.0","source_stated_rank":45},"run_fingerprint":"8114efd625760b47d3d27a7b87a3c306902e918b40edc911b3c6b1ad07887775","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"49449f35b81df3494d288070c6a8ed7e3d9398098fc8a0ee92a97e29a7cb170e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (Max)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b4875e94d18b050b87239fa17e624b0ae70852c2826cbe5d3cc85ecb71209dcb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":89.0,"score_display":"89.0","source_stated_rank":78},"run_fingerprint":"f584431ea8fe97ff9252671d744f041c4e75e2ef9ac763f5f96d6098542f8d8f","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a0fe37d25fba8339eea179d246be3be65f7ab79f022d71c8b841bd0cfb54f329","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (High)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"78472727b3932e170c5c98599e77e11d5105508036e4620bd4514ee9ecd81408","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.0,"score_display":"87.0","source_stated_rank":89},"run_fingerprint":"99ba5ab495c8d8216eed182244af12f7abe2bbb02a6c1525cedf31b5dcf0a2d7","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"13f6ad9184480658943e0aa9c4987762620d922ef1ab95f7009068204bb96010","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (Low)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"81758a49f30955e8252eff98966f1379f4c1309a8aeb0ff048f43437aead05c7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.0,"score_display":"84.0","source_stated_rank":106},"run_fingerprint":"46f80041d83b8a1e58c8189da06769a70298a91942cbeaca39d9b2677488c84e","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_dd089538c848df7e5e4d1310","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"918965e5eda6ade9c69c0ad9a0702d412a37cf8a5db97224d76b6657d37461fb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (None)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"816d28d96a45e69dc183193c87bcd11c42daa12428abced1bdb05598221a48f3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":11.833333333333334,"score_display":"11.8","source_stated_rank":247},"run_fingerprint":"b73c0bdf0f340fb203bc578e16aaa550f14f65a195d46e370ecdd73cbfee4fd0","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e29d4b74c396a06029c21050","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ae4ae0d28cfb497825937c974c5e4557436864f85aa487555b30d60e2edffac4","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.009884,"date_is_proxy":true,"latency_seconds":202.391,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.412},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":256000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-flash-0731"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"d2320239d7b8e370d88bc7a614508d4f35fd9131f334702e21179e4dee630e14","result":{"confidence_high":91.56752,"confidence_low":86.03247999999999,"sample_count":500,"score":88.8,"score_display":"88.8","source_stated_rank":11},"run_fingerprint":"4cafa9dd38331666944cde2495aaa3272538a96ac5905665abea1b4bb4175649","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_232b90e5b13ff602749b9048","split_or_window":"qwen-refined-vendor-comparison","unit":"percent","version":"SWE-bench Pro · Qwen refined comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":39,"at":"2026-08-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-27","status":"fresh"},"measurement_id":"214b0520ef03b7e4a1a337476c70b4d4a78a9c66f5cc712bc87d7c539f1c1573","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published. The score was evaluated by the report publisher, not by the model provider.","comparison_warning_code":"report-date-proxy-third-party","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"README · Language table · Coding · Agentic coding SWE-bench Pro · DeepSeek-V4-Flash-0731 column; note 2"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy-third-party","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on its corrected/refined SWE-bench Pro task set; temperature 1.0; top_p 0.95; 256K context; exact task count, run count and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-27","source_content_hash_method":"raw_bytes","source_content_sha256":"35ca37ccc366f1ba478dab33841a2c0c18ce53fd62f291ca05341f7728b225b2","source_published_at":"2026-08-27","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"0432fd35ae40b691fb131efec38f0138eddcaf55773ed70658dd2cd02ecf21f4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.0,"score_display":"56.0","source_stated_rank":null},"run_fingerprint":"4bcb20e11c4d94fb236bdf93a61f9614b8e4c27913b2d7a8dd7b2f7e58f521c3","source":{"content_hash":"a8132c0d08dcb8936d1a9078fbf74a39072aa1d37bc0b3c112c58e2308514ae5","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next","id":"qwen3-8-flash-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen Community License model card; aggregate score facts with attribution","locator":"README · Language table · Coding · Agentic coding SWE-bench Pro · DeepSeek-V4-Flash-0731 column; note 2","name":"Qwen3.8-Flash-Next official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_126fd03ce308e240a956d81a","provider_config":{"source_id":"qwen3-8-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":6,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"third-party-vendor-comparison","verified_status":"third-party-vendor-comparison"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3ddd4f24531b0303e8597e1a3ea815b2ac08540987ab4881e63661f819facc03","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731","source_effort":"high","source_model_version":"deepseek-v4-flash-0731_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"chisel","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"f9634c556a75573db6829b966ea95231cab463e5f52b2c35a89959cfc61962b3","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":18.8,"score_display":"18.8","source_stated_rank":34},"run_fingerprint":"1b3d0b6ae926e779542b551cc265422bb0caa9f5a1ce8d91af52fc19d1c158fc","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f7544ced21a9d223f7cba85a7a30f5e32b976b6e34ecaefb716c48255e779f61","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.373112,"date_is_proxy":true,"latency_seconds":1663.091,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.35},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":256000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"high","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-flash-0731"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"83562481c5d7bf299134961682acdba9575a4e6e3f73b17885f0b01c9fa54e62","result":{"confidence_high":37.328,"confidence_low":28.116,"sample_count":null,"score":32.722,"score_display":"32.7","source_stated_rank":36},"run_fingerprint":"4c89430feeacd914ee2646217a060c02c4c56ce33a63b430927452f942fa80cb","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fbdbf3fa8f8721e28ef1cf0d2209cc2fd9c2225376b689c80e9b43eb88a99d4c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.020251,"date_is_proxy":true,"latency_seconds":126.61,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.968},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-flash-0731"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"4a1620a8665ac2dea4201dd1c4b11e57c313a065e37ad89f648dd8de17b04ce3","result":{"confidence_high":89.16127999999999,"confidence_low":85.36672,"sample_count":null,"score":87.264,"score_display":"87.3","source_stated_rank":17},"run_fingerprint":"b4bf3412f959ff45427d114a457ac5f7f61e092c7526b9b25a60f0199a3955a7","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"77f25ec52188730ab220b20cc19859bc51fa9a8f8b65edc7269806f6b2ad6b78","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"13c6184b21680ba605fc94a5455bcdeb451e82d3a9e2691e8a8aa4708d51f2ba","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.88425925925931,"score_display":"49.9","source_stated_rank":94},"run_fingerprint":"446bdd18c068dd0a2f7ddc136d2cd0108cb765ea6f05c19fceb7cb1c5714d6eb","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cf2e2de14f9b40d0d4a30b6a92598d4e46c5d4b29b2888c1d426125726de602b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-31","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · scicode"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash","upstream_model_label":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"8b03a27a1193cafb70a23e083888973965d39a7b234d02221ff608393ec9f704","result":{"confidence_high":56.079087,"confidence_low":44.606216,"sample_count":288,"score":50.347222,"score_display":"50.3","source_stated_rank":null},"run_fingerprint":"9e6dafa6ebd48ccd90807ed9f0c29406ae6c6a7f46f8fd40bd5e69bac35ee68e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3ed8095903f1f7fb3b2c67a6db38ef0ef64676222701a1a89d72f72869f22f55","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.019526","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"deepseek-v4-flash-0731-max","source_effort":null,"source_model_version":"deepseek-v4-flash-0731_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"a6ed17a92b410aab877736161aa7f4ef805d5fca3b89423114f840c6c590c7fd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1306.08,"score_display":"1306.08","source_stated_rank":28},"run_fingerprint":"6113c8ea484d775452d945fab9955b954011c25cf9de3177860a7667f7cb356a","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"044397d56222946e1fd484e8a5fc6645f46a12335441e3d7add4e75eb3f46e47","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.905592,"date_is_proxy":true,"latency_seconds":2241.572,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.442},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":384000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-flash-0731"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"5622b9070a65d6f786e84e8099047b846b523ec058f98c6548f087f4bb036638","result":{"confidence_high":81.48232,"confidence_low":67.98968,"sample_count":null,"score":74.736,"score_display":"74.7","source_stated_rank":30},"run_fingerprint":"fdbe9514acfcaecd54ccc11a91b4397a30dfaa7f6cf7954a2a7acea867c1b3df","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"117572d11833cf7718ddfe284a13f0dca9f20ded3852ddb73dddc7a20a516314","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-flash-0731_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"fe5398199e5b4ae0f55fab2a4bccdec2bf4197cd7613709660eebe0894a7cc51","result":{"confidence_high":1589.75,"confidence_low":1570.65,"sample_count":4723,"score":1580.2,"score_display":"1580.20","source_stated_rank":26},"run_fingerprint":"b58e30c7e193282e9a7fd87c9172c9c40dc653bbf5e62dd150277309d0141360","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"09423b44a8155b6477cbfc15218068924f9f7b2a72345de7c8d6853d14bb36c3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-flash-0731_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"b28eccd1504c39139f2e046ff9f2e5210909468615c499a8547165b56f927d6d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.96000000000001,"score_display":"63.0","source_stated_rank":44},"run_fingerprint":"f0a7b708a986dcd5b1dde3053483b11ba446238ded7328abe775a85f94918dee","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"868d36269c5920f1251bf68a79729c35c3aa53caba5a0e721ad051da2730e33e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-31","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-flash-0731_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"37f01f9608c5bb73a710ae194f5beb8d52f5efa53279ec1f6bbae7dcf8e416e4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.05,"score_display":"57.0","source_stated_rank":57},"run_fingerprint":"042998e1aa5eb10e1ccda8f5ea26a942981af24fe0748ab01cb325a8dc19276a","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_e2fefa2f41df7df75dc708d0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"62ffafaa89c06d4b41747b196fc995f5b569513d56ab3d8d77b61eee9c374a4c","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.021068,"date_is_proxy":true,"latency_seconds":683.03,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.35},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":false,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-flash-0731"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"e82d6e3e97b8c1641ba8acfaf04ec52b9b9c18442c5e10be491f51d6b1cfbdbc","result":{"confidence_high":69.687,"confidence_low":64.395,"sample_count":null,"score":67.041,"score_display":"67.0","source_stated_rank":36},"run_fingerprint":"8c538eda4a0ebb96002a9b4d4fc47f32b5cbcb553cd0a89612de97e2acef5079","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7379d89dcae12f3817cbf811","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"430bff65e01c16af8024abcefc0c04f7414d97e464d00fd1c2ec8894413d7ae0","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-31","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · terminalbenchV21"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash","upstream_model_label":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"a20bf3aee4754f91b4224e39b108e63d917c7ba09897677820ceff4abb0f1a05","result":{"confidence_high":85.885313,"confidence_low":69.046958,"sample_count":89,"score":78.651685,"score_display":"78.7","source_stated_rank":null},"run_fingerprint":"c2e76bdabe5569790e20d1aa7435fedaafda61a84735ed3dbbcf086b5ec0065d","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_83e2a53bddae6e68d7ed56ed","split_or_window":"provider-reported-terminal-bench-2.1","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":66,"at":"2026-07-31","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"fresh"},"measurement_id":"4e1279878fa54068a01597725b119c04ca28dee75c3bfe9d705dfdb32c7971b0","metric_details":{"comparison_warning":"Different agent harness from the Terminus 2 run; both measurement series remain stored separately.","comparison_warning_code":"terminal-different-agent-harness","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table and note 1 · Terminal Bench 2.1"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"terminal-different-agent-harness","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"official comparison score; exact effort, sampling and harness not restated for this model","reasoning_effort":"reported","result_published_at":"2026-07-31","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"terminal agent toolset"},"run_count":null,"scaffold":"not restated in comparison row","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"2856f01ce5091a1b0fbf9afd2ac976cbff4761be067a817d79b63afd325fbd4b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.7,"score_display":"82.7","source_stated_rank":null},"run_fingerprint":"4fe037c0f8b6df6c0f30beb35c8bd8d6343a95196eda5b3b3905972aa64cc9d1","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table and note 1 · Terminal Bench 2.1","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bf407507bedf02addf42d3d4","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":4,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"secondary-re-report","verified_status":"secondary-re-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c881527f5e6baac66ee10bfd347c8203fd3cb2805df47bf0581db5a0d1a2132e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-31","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · tauBanking"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash","upstream_model_label":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"fa30a46d3c4e2647076ffb19dfd97ddfbc7fbf8e0338826a7b14058b3ca2cdca","result":{"confidence_high":49.330945,"confidence_low":30.240978,"sample_count":97,"score":39.381443,"score_display":"39.4","source_stated_rank":null},"run_fingerprint":"124a3ce37f95f81e602245ea38783425171cab20e1c67242d6966eb668974357","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Reasoning, Max Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aab40d48e51b228cba256824fa787fdcbbd192ccbad1cf0ddbba815c0396c5a8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-31","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Max) · gdpval"},"model":{"id":"deepseek/deepseek-v4-flash-0731","name":"DeepSeek V4 Flash (2026-07-31)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"deepseek-v4-flash","upstream_model_label":"DeepSeek V4 Flash 0731 (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"24f497e4b025755fdc6504c09ad34332b44cf5aaa41b4d0d7878064573194d82","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1426.57,"score_display":"1427","source_stated_rank":null},"run_fingerprint":"449eba4ae47099c17f079cacd617d24a4a18c6f41e55c6b79a29c4df8ac09174","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash 0731 (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_58929078b91d4914279ad26e","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a5a8ea7655bb75ce4bf58ae96d567e8b9c26524580d8fc0f2b82b774f381bfdd","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-05-19"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 3.5 Flash; model release: 2026-05-19","source_accessibility":"API access","source_model_name":"Gemini 3.5 Flash","source_model_release_date":"2026-05-19","source_model_versions":["gemini-3.5-flash_high","gemini-3.5-flash_low","gemini-3.5-flash_medium","gemini-3.5-flash_unknown"],"source_reasoning_efforts":["low","medium","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"952a8a5c8bfcebbbcdc65a173dc244e094fc16df4701bc125da3cf45569e32e9","result":{"confidence_high":156.62,"confidence_low":152.53,"sample_count":null,"score":154.51,"score_display":"154.51","source_stated_rank":33},"run_fingerprint":"87f0126f2b07d43c122aa51c6480af2a29b6763acca6a0cf73a97ade542fe99a","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d3b2abb9ad25b246a9b715e1","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"98308c612b5f8aed54d20dde5d86c92da5af34ddf3de3030dd769ccd979987db","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · intelligenceIndex"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-medium","upstream_model_label":"Gemini 3.5 Flash (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"ece21402b7db9ab1551ff862ef33e549dbbc906e6b69e836cae30d4ee0ff778a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.632587,"score_display":"33.6","source_stated_rank":null},"run_fingerprint":"cb5be9a0d04625357d892d13a2f177fa35a20912176d03556f5f7cbe336720a0","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_667b754faded84c3180149c6","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"34abd53b2a87dbddfce3c6193d2ac8c335504824993d1e941ae89f2dc567011d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · intelligenceIndex"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"minimal","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-minimal","upstream_model_label":"Gemini 3.5 Flash (minimal)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"21117252487d8e893551c1567e1270c8351f1300d22d7b81ee4362b2b8d239aa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":23.848791,"score_display":"23.8","source_stated_rank":null},"run_fingerprint":"b5f6597ca42e531736467d08f4cd6bd760039b331889dec3d0ab13c7f57aaa34","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"49f644aadd8d129fa85c8d1f75dd74d05316fede6adc96c599d4f3b1cdfe8fed","metric_details":{"license":"Proprietary","variance":3.9557738091983055},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1481.2962837695752,"confidence_low":1473.4998891689881,"sample_count":48420,"score":1477.3980864692817,"score_display":"1477","source_stated_rank":28},"run_fingerprint":"1a6476797dbb9b75d6e4dee7ee2dc03f7edb2029f4bf882810152c0d5e401c36","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1ffdd21eaa27393de3bc3b15aa35d95871b21cb52b9dc47bdb6b224b76f197dd","metric_details":{"license":"Proprietary","variance":4.485072456421005},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1480.2419340915783,"confidence_low":1471.9403146955397,"sample_count":47187,"score":1476.0911243935589,"score_display":"1476","source_stated_rank":31},"run_fingerprint":"5dc40a85beeafad8a6b9a811f251649a459f9bdedff01658a0fcd3d6e86b8a24","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_667b754faded84c3180149c6","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ee75073dfe9f96462eba7415907c0c2f8c92ae02eff6bede8e9cf6d6c4919f7a","metric_details":{"category_scores":{"Agentic Coding":48.989666666666665,"Coding":78.1845,"Data Analysis":64.85733333333333,"IF":75.6,"Language":84.58433333333333,"Mathematics":88.24375,"Reasoning":82.00475}},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.6377619047619,"score_display":"74.64","source_stated_rank":39},"run_fingerprint":"fe71b1365955d3e6b8ccaf12cff0eab6da7b803b0ffec8b691cb307de1c179bc","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":139,"at":"2026-05-19","basis":"evaluation_at","evaluated_at":"2026-05-19","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"797fc2e0417b47624835cb25f38009e532d955528b95007dd153ca50a8b0007a","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/google-gemini-3-5-flash","source_model":"Gemini 3.5 Flash"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f1c14751cb135ea7742188a867c1ba4dd19b3da37521600b924d9d938272dc0a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.5,"score_display":"42.5","source_stated_rank":11},"run_fingerprint":"a8b4ed847af0e912a3f44e915ab3525871c03bfee7594d7dd8b12ce74f152abc","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_04a576d1beb0fd7c718f6932","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c62bab24fb1d6829f20f01d802d7200d387b944b1450548295fddb7e5701e685","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · hle"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-medium","upstream_model_label":"Gemini 3.5 Flash (medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ab03824db671c6d94897a344c9a9308acfa6123fc7a3526b1403f986d011dc68","result":{"confidence_high":43.425856,"confidence_low":39.274079,"sample_count":2158,"score":41.334569,"score_display":"41.3","source_stated_rank":null},"run_fingerprint":"963f4bfa82beb3aae899062359d6ccf45ff8cf09234f8a4e42547b2c1741eab0","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_667b754faded84c3180149c6","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"750b782bfc0ed40d6cdc59410746b52674ed41cbce2542745299c352be31af41","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · hle"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"minimal","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-minimal","upstream_model_label":"Gemini 3.5 Flash (minimal)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"89caed3f0c1a478eb1bb2cc535202db2117ace42377d87092e168364675c832d","result":{"confidence_high":25.94582,"confidence_low":22.339014,"sample_count":2158,"score":24.096386,"score_display":"24.1","source_stated_rank":null},"run_fingerprint":"f79641a31babd1a25e494ed474044644c1dcc3f7b9a441ae7bdfdf6f7d271234","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:30:35.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:30:35.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4ce0f41f1b73954b1f1f6806c0bfebf978f17873269440f4b6b19ff6653562d8","metric_details":{"best_score_across_scorers":"0.662","model_release_date":"2026-05-19","stderr":1.4965960710224384},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"GJ6YsSuzyRD9YXe6SBwvsu","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGJ6YsSuzyRD9YXe6SBwvsu.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/GJ6YsSuzyRD9YXe6SBwvsu.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e862642b958218745c8c4c30f3ef5e81069ee89a2112ae7eeb5a3627c1e7c8a1","result":{"confidence_high":69.13332829920398,"confidence_low":63.266671700796024,"sample_count":1000,"score":66.2,"score_display":"66.2","source_stated_rank":11},"run_fingerprint":"4932c0a55cea93a3eea194877789dd80f1c1ded5b40a6388521bec5e4b99112b","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":126,"at":"2026-06-01T11:15:19.059144Z","basis":"evaluation_at","evaluated_at":"2026-06-01T11:15:19.059144Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"64c114285b77f89c7a646fc04d7f49f6794d3ff6b0f835aafbd86c7d9ab3a503","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.5-flash","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cd5e6afc7129a01aa95c31ef50ece396cc24f60f1750474fd669239c16c19f76","result":{"confidence_high":73.28609008158443,"confidence_low":67.53637732062217,"sample_count":null,"score":70.4112337011033,"score_display":"70.4","source_stated_rank":6},"run_fingerprint":"2601ad85497f8f0c77f6d1cd3e0a5040ca8b4aff6e40a9c65f63121539b68b28","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dd26373552419a246f3ae988","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ac9c07335cbc97e93f3d5856c77cb66088614856518ab6ba284cb5fcb344353b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.063842,"date_is_proxy":true,"latency_seconds":27.655,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.462},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":91.414,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash","zero_shot_accuracy":93.939},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"ce8104fa5f0b38dad7afdf9c0f96da59ee69545913d7b037ab92decc9531f740","result":{"confidence_high":95.54152,"confidence_low":89.81048,"sample_count":396,"score":92.676,"score_display":"92.68","source_stated_rank":14},"run_fingerprint":"7f5a01aa15616dfb67f0f9a424d2cd9ace04308d15eb1daa5e090f9edd916ccf","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f4c4be193a45fe986a2d4dbe6d48fc4a31f6bfd0b600ae51e5987badd49f0e15","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · gpqa"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-medium","upstream_model_label":"Gemini 3.5 Flash (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d3419814c3bceab0d2e0fabd3e28924e3f9127377756f2cfc77f4fcb03f5add4","result":{"confidence_high":95.12173,"confidence_low":87.517329,"sample_count":198,"score":92.121212,"score_display":"92.1","source_stated_rank":null},"run_fingerprint":"151dd22effc1932c9b1752693aa45db88277b0a61ce30329a345c6b454e2221e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_667b754faded84c3180149c6","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9522ed391c59e8371e8dc16a4975541a13520d6627ef245c8f33a158d0966d4e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · gpqa"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"minimal","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-minimal","upstream_model_label":"Gemini 3.5 Flash (minimal)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"cebd629fe70614029856d1eec80c4ad4d1e9058d6e61bc61d1e8eb173bdd1921","result":{"confidence_high":87.44377,"confidence_low":76.963171,"sample_count":198,"score":82.828283,"score_display":"82.8","source_stated_rank":null},"run_fingerprint":"28191482145fa76618aa3b058967bb9e0687109f3a3d32ce3b115771a909dae9","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":135,"at":"2026-05-22T16:02:44.000Z","basis":"evaluation_started_at","evaluated_at":"2026-05-22T16:02:44.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8eae7f5af1b89fba9085e8f6476ac70d63d97807d0143ce06d401a1f2c067daf","metric_details":{"best_score_across_scorers":"0.928030303030303","model_release_date":"2026-05-19","stderr":1.6387002835275906},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dYYtcwENFRtm9jZTDA2p9C","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdYYtcwENFRtm9jZTDA2p9C.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/dYYtcwENFRtm9jZTDA2p9C.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"489f9c20d708b593773ed6b402b34749703371cc669de91951457d71a8453823","result":{"confidence_high":96.01488285874437,"confidence_low":89.59117774731622,"sample_count":null,"score":92.8030303030303,"score_display":"92.8","source_stated_rank":22},"run_fingerprint":"85861a5e432fd4b5d3619bcb4fd55841cc6e4177854616928e4c556bc826b2c7","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:30:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:30:55.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a0efd4e0ac18fcc2645a2584e3f1bce78e9413cad7985a190aa9d7c17b3ed01f","metric_details":{"best_score_across_scorers":"0.8888888888888888","model_release_date":"2026-05-19","stderr":2.239078763821682},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"eiTYduCDNq3HgwTg4ZhEw6","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeiTYduCDNq3HgwTg4ZhEw6.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/eiTYduCDNq3HgwTg4ZhEw6.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"19fc68f663ba51c814221a14223771ff10e9f291467a898deb99db5bcc101b73","result":{"confidence_high":93.27748326597938,"confidence_low":84.50029451179839,"sample_count":null,"score":88.88888888888889,"score_display":"88.9","source_stated_rank":53},"run_fingerprint":"e737c86bf338f8cdc54b6fcbdb81f899c55b832555132f08b4aa22cf1cee04b0","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a34352f21e8e58806cb716f6","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":82,"at":"2026-07-15T03:14:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:20.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bf76324a610c567dac152c4cee386f50b2d3d6e395cfc5cad9765310fea2ab38","metric_details":{"best_score_across_scorers":"0.8636363636363636","model_release_date":"2026-05-19","stderr":2.445015597318985},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"QDZRoAVaEDtdS6nECEeE35","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQDZRoAVaEDtdS6nECEeE35.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/QDZRoAVaEDtdS6nECEeE35.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e36403b5f29fd6192abc116437d707adee39492ce64926dc5b3becdc238c1760","result":{"confidence_high":91.15586693438158,"confidence_low":81.57140579289114,"sample_count":null,"score":86.36363636363636,"score_display":"86.4","source_stated_rank":76},"run_fingerprint":"6ccc9c79160a2986cf743239cd020d2e7c6606fc02e686afdb5d52ad99a0911a","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":98,"at":"2026-06-28T14:53:14.329383Z","basis":"evaluation_at","evaluated_at":"2026-06-28T14:53:14.329383Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e86429c4f31a8e2e1150a9c97299064dee99910a2df88aaba3a7c60d09eca993","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.5-flash","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"714d563aafe0fb22900a91c2c034336ab325fc450cd414d9d1cff0256bf38b3c","result":{"confidence_high":96.10995513204666,"confidence_low":88.73852971643818,"sample_count":null,"score":92.42424242424242,"score_display":"92.4","source_stated_rank":4},"run_fingerprint":"856636c438230b4f2afb9c359589f58dee81f01a6862a5a4f3ed2419ee7c3f77","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c2a65ca4187011be8ac7f62c","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4fe1272000c3f394f7812d6e2d6b747b1bb75a47e390cc7fd1579cfc4a486e46","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.02498,"date_is_proxy":true,"latency_seconds":26.262,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.306},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65536,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8b095e349f979bdae01fbd9978a0734c1a64013899a24ca3845c0365d8057bf4","result":{"confidence_high":90.11476,"confidence_low":88.91524,"sample_count":null,"score":89.515,"score_display":"89.5","source_stated_rank":10},"run_fingerprint":"00a035d1248d7c912c2cd79a52b213a03baec4fc32c63a2a723748ea87c717e4","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":133,"at":"2026-05-25T13:10:18.000Z","basis":"evaluation_started_at","evaluated_at":"2026-05-25T13:10:18.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ec08b3a3ccf0c4eae51214414812da9d4d6220e6eaec6afd3925ccea6149fa58","metric_details":{"best_score_across_scorers":"0.9555555555555556","model_release_date":"2026-05-19","stderr":2.6696111689702042},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"eYmgjYJfM7Utm46nfoRfZ6","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeYmgjYJfM7Utm46nfoRfZ6.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/eYmgjYJfM7Utm46nfoRfZ6.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f5fdf83eb3d4f1fc9c2086fc3f859867e4070da47236d1a1788e969ac7c2d436","result":{"confidence_high":100.0,"confidence_low":90.32311766437395,"sample_count":45,"score":95.55555555555556,"score_display":"95.6","source_stated_rank":38},"run_fingerprint":"bd41bc5b31306ccb6dc818f1ac863059944197f169d05067e8d9b4720bc2ff96","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:34:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:34:49.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a70296b6e66c97a85fe1a70c0784a7e40747adbc907de59bf69c1d79ef47c79f","metric_details":{"best_score_across_scorers":"0.8888888888888888","model_release_date":"2026-05-19","stderr":4.737793696791343},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ig7gQ23WRms9SQLLNPtbTi","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fig7gQ23WRms9SQLLNPtbTi.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ig7gQ23WRms9SQLLNPtbTi.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d7d9880849f27c1589d260eb5f4b4ba9550e8532a405b175e264f5ee0f48cc6d","result":{"confidence_high":98.17496453459992,"confidence_low":79.60281324317785,"sample_count":45,"score":88.88888888888889,"score_display":"88.9","source_stated_rank":71},"run_fingerprint":"900efa613afac3515800c3a0d43a3069578b483711698c26f8549b19c8653b47","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a34352f21e8e58806cb716f6","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:20.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6bfcb51418f8b616b7f48ec3a1852a727e3170b1f7a02c58c6e2af1c09a7952f","metric_details":{"best_score_across_scorers":"0.8","model_release_date":"2026-05-19","stderr":6.030226891555273},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"JuM7FArkmhLqqKBqudySsC","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJuM7FArkmhLqqKBqudySsC.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/JuM7FArkmhLqqKBqudySsC.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d88c317df3b832941ffd8ddaa68f2a99584951790ea522aa2266f466c2756a48","result":{"confidence_high":91.81924470744833,"confidence_low":68.18075529255167,"sample_count":45,"score":80.0,"score_display":"80.0","source_stated_rank":110},"run_fingerprint":"e8ae1be5c113bae40d5983ba4e932f3a5d93d60730a33e0d37f184e8b0bf2e6c","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T17:32:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T17:32:38.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e53f4879035dfd64dcc1680eef97eef1bff3f9cdc8c9cb6c8c4539c1bcc208c0","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.6280701754385964","leaderboard_private_problem_count":285,"model_release_date":"2026-05-19","public_example_count":10,"stderr":2.867975375107462},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"72DQVwbYiQW5pjhshjBaGU","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F72DQVwbYiQW5pjhshjBaGU.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/72DQVwbYiQW5pjhshjBaGU.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.5 Flash","thinking":null,"upstream_model_id":"gemini-3.5-flash_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"daa218ae0b434ce51d940177b4aeb7f1dde4b11fe1e481fe1df08e75d4d6e5b7","result":{"confidence_high":68.42824927907027,"confidence_low":57.185785808649015,"sample_count":285,"score":62.807017543859644,"score_display":"62.8","source_stated_rank":34},"run_fingerprint":"e1d23cfac672957ddf38b7bfb6f158e12ca46a79dad3c0bbf428c01d201e58c3","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5ec81b1940df20ecca523a67fd4e525803b23706019ee585a9d861bbc2385ada","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.8503,"model_release_date":"2026-05-19T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-5-flash","model_type":"CoT","upstream_model_id":"gemini-3-5-flash-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"dbe38c082ebd2c37cdc880c1706306279374fbeff0205fdba64e0d763061a9c1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.08,"score_display":"72","source_stated_rank":50},"run_fingerprint":"7198be0c0ad7abe50cd1032347ee95cdd8a502d999d9090167235761693e5f72","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6df5f9625e30210e0312cbbb63597ca760df50a30bf5d944941e8777c685ac98","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.1074,"model_release_date":"2026-05-19T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-5-flash","model_type":"CoT","upstream_model_id":"gemini-3-5-flash-minimal"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"698fe7d80d0786668c3741764220a023ac19d6ec2183f8350e5243a585b0546e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":8.89,"score_display":"9","source_stated_rank":146},"run_fingerprint":"f24896a652c72269562a884ca7792da9c22acb64783e0f65351042a0f6a71204","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"29a4e7de5de637131550456f8cb1f2bc5074dd8b1c3c58c4c108b62547766d00","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.8503","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash (High)","source_effort":null,"source_model_version":"gemini-3.5-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"380295c0d837152d14e6974484838053cc87b981b142d285ca8673905709b3d1","result":{"confidence_high":76.46238,"confidence_low":67.23136,"sample_count":360,"score":72.08,"score_display":"72.1","source_stated_rank":53},"run_fingerprint":"8e23890dd5ea2a5256dd86e5b863b23a05bbb14fcd4a15c28c2c5045dc86ce65","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a05614c235a2cab62e449007be9295bab8ed980a0eccd47bb12c6e6e4b5baf60","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.1074","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash (Minimal)","source_effort":null,"source_model_version":"gemini-3.5-flash_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f8f2e53fa9cb3a6eba7d4cdbcb4c320962b6eb5e6baeb084dd2d9d5efc72ad30","result":{"confidence_high":12.280475,"confidence_low":6.36764,"sample_count":360,"score":8.89,"score_display":"8.9","source_stated_rank":142},"run_fingerprint":"1670abeabdec9f6627091c8c41bc2c54cb99f96bb323f5076e6da670eb013171","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_3442b9e1c78d3919dff3a62a","split_or_window":"semi-private-validation","unit":"percent","version":"ARC Prize verified semi-private"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":139,"at":"2026-05-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"f3b112d81991452c6b6607196e0da120131ed7f9879e23ff24558b0432841a06","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · ARC-AGI-2"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"ARC Prize Verified; Google Model Card re-report","reasoning_effort":"reported","result_published_at":"2026-05-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"ff0a67497c6d2377c884521186b2e9dddc4cf6e001e044d02319150159a2f674","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.1,"score_display":"72.1","source_stated_rank":null},"run_fingerprint":"45d0a712a5b4ee2f73b854abe19893acdce4f9be7ab7922f6fddf02540a94829","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · ARC-AGI-2","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e45acdfcfb596c07e0973eed","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T17:32:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T17:32:38.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ee8bf9bf41bf603501c77c7c4db4b24eaa7f656d36eb2dd74ebb8e363e60a500","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.2682926829268293","leaderboard_private_problem_count":41,"model_release_date":"2026-05-19","public_example_count":2,"stderr":7.005564203095157},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"cXn5zzns6VFwbTgotDwZjV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcXn5zzns6VFwbTgotDwZjV.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/cXn5zzns6VFwbTgotDwZjV.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.5 Flash","thinking":null,"upstream_model_id":"gemini-3.5-flash_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"61b9e7719f4d628014c835305095e2f34f2afa822735c4212159bb6315897cbc","result":{"confidence_high":40.56017413074944,"confidence_low":13.09836245461642,"sample_count":41,"score":26.82926829268293,"score_display":"26.8","source_stated_rank":37},"run_fingerprint":"2bd009528c81eb78b539b5f6367228464ef5b1fb9808d1c17dd3ac8bf42eec28","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"64f09d5df60dd3c7b615ba408b395f2308ae0c85f0158d9b2cd00ced8815b5d1","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.025155,"date_is_proxy":true,"latency_seconds":12.226,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.774},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65536,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"fd0665812d08349c19b203f3afafa562abb7f55e3fda3de5c9f89a353eb9df15","result":{"confidence_high":89.78304,"confidence_low":86.74896000000001,"sample_count":null,"score":88.266,"score_display":"88.3","source_stated_rank":8},"run_fingerprint":"bbecc05af75743a54c10cd7d81007b24556d96b3eb47c552c435c2e2e0b2af70","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ea17360bdfe585a7f722e71bec5c6219519ebf30f70d9ff67b6db3423dfef484","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · mmmuPro"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-medium","upstream_model_label":"Gemini 3.5 Flash (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6cc6e19a9242c771c21b3213ed6c4b97f0d6a163404f68239517efacc35a32a1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.872832,"score_display":"83.9","source_stated_rank":null},"run_fingerprint":"d3ea58288a37fb2f6311bb82f1767505e94ea094ce9dc7156c0b59c417415d17","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_667b754faded84c3180149c6","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1c728933d6fda49c2c77148bc91d439a3537b6947f927e75e032e692b39b001f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · mmmuPro"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"minimal","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-minimal","upstream_model_label":"Gemini 3.5 Flash (minimal)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2aa503b10b35d1fdc79197c7debc181fd7a68b97ea6bf100f8418f08a014282f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.115607,"score_display":"80.1","source_stated_rank":null},"run_fingerprint":"0cc7bc77f30e20c66837d02aed2fd961ed6ed22f34d81042dacabc1d1586bbad","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_39a5bff9a48a29bea5ce760d","split_or_window":"standard-no-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":139,"at":"2026-05-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"56a60b85692e9c24819c1ed47a555e6c51702e51f71cfa782861ca62f0c995e3","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · MMMU-Pro"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider evaluation; standard and vision average; pass@1","reasoning_effort":"reported","result_published_at":"2026-05-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0fe020a8765804804ad5ddbf01716ecad56c8a711c41eae06fffe67f7933b97c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.6,"score_display":"83.6","source_stated_rank":null},"run_fingerprint":"8fbef2305283f68b34ad0f714e55aae59f9d55a923fe61daf2c71699f865ee2a","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · MMMU-Pro","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e45acdfcfb596c07e0973eed","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"00884df7dee73458458babdcb88cf3f7a032a84c9d7f74876b8a2634c29ab756","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash (high)","source_effort":null,"source_model_version":"gemini-3.5-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"725c3e7f4a8f76dde2e19f650dce7786f8a2fd295e4d45f4b25c7484ba5a55a7","result":{"confidence_high":22.919817,"confidence_low":7.149632,"sample_count":71,"score":13.1428571428571,"score_display":"13.1","source_stated_rank":83},"run_fingerprint":"3962ad75f669b70977061a3aa5e202d5ddad6da2e0c76737a4dda2a571b85daf","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b877fcedaf7e0b8d7449adfdf5c3f885afa995a0c10f19bfc320e83b4ca1b1b1","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · critpt"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-medium","upstream_model_label":"Gemini 3.5 Flash (medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"fe4fa2782d13a57e119f91514079e6626faa1d8f98c2f821fe20218cb583b5bd","result":{"confidence_high":20.275857,"confidence_low":5.511234,"sample_count":70,"score":10.857143,"score_display":"10.9","source_stated_rank":null},"run_fingerprint":"3083376fb9dd8581b875d9a217f9c981419881f9f5430ef6c46e2026b08de7c1","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_667b754faded84c3180149c6","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"34aeddf75921a0a2c842213a8e344c80a858215ee2b9659450ccc6485953c23f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-05-19","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · critpt"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"minimal","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-minimal","upstream_model_label":"Gemini 3.5 Flash (minimal)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"742396565f26ebf43916fbb80c6493538b2b01b5e791a299d45c6d5d0f61577b","result":{"confidence_high":7.658369,"confidence_low":0.252618,"sample_count":70,"score":1.428571,"score_display":"1.4","source_stated_rank":null},"run_fingerprint":"ecfc9f5cd93ba0e1482c41eb5a85a4e4196bb7f21bf816818be552c3bfa58dba","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (minimal) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ce2fbff7c48c15b9280a8c73a407f676888bcac80fa818cfe4e4d3e24d5bfc3b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.5-flash_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"34c405ecef08f9fa68202ed884d799037870c7441b23f79acba78c81c9c5ff3b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.7,"score_display":"76.7","source_stated_rank":6},"run_fingerprint":"ee796993a2214dba18fc64f32210d7d63f1ee774b4da9e5ae81f081bb50745ba","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_712ce459e409b423c301b510","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":130,"at":"2026-05-28T10:10:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-05-28T10:10:09.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3be2d92fca17b63045d50e2821c30d33a46223e53546d77485a1a317bf47ad77","metric_details":{"best_score_across_scorers":"0.5","model_release_date":"2026-05-19","stderr":5.02518907629606},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"imPXZhfTfq9Q9G2kq3stdg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FimPXZhfTfq9Q9G2kq3stdg.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/imPXZhfTfq9Q9G2kq3stdg.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d38237ad322322aa11edc43fea44f68e917f0f858b08345bdff731b96c505e0a","result":{"confidence_high":59.84937058954028,"confidence_low":40.15062941045972,"sample_count":100,"score":50.0,"score_display":"50.0","source_stated_rank":11},"run_fingerprint":"ea0e777b88fdd086eef92d29ccaf236ba5ef383f34c0c5b927c6f8b3bc49c24c","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:34:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:34:38.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7b31c7f6fa2167b9a1e290cc3ecb03102273adb53048339ffa8919835ef6f8da","metric_details":{"best_score_across_scorers":"0.45","model_release_date":"2026-05-19","stderr":5.0},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dQhoYvW3Wz2eZSZ89K6igd","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdQhoYvW3Wz2eZSZ89K6igd.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/dQhoYvW3Wz2eZSZ89K6igd.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"2fd32af6ba2afbbd98b8848f2d494fd922bbd71b854f598a67e516f1453851f2","result":{"confidence_high":54.8,"confidence_low":35.2,"sample_count":100,"score":45.0,"score_display":"45.0","source_stated_rank":17},"run_fingerprint":"60ce50a99bfcf8152d08265def32cbe98d85e9e58fe5ddf6f777c767e040f548","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_a34352f21e8e58806cb716f6","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:20.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1b3ead424602fa61d6f67a904656b3ac7d196ec1e32cab8c6aa8052a027188d0","metric_details":{"best_score_across_scorers":"0.43","model_release_date":"2026-05-19","stderr":4.97569851956243},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hYeysTWbJLkxwBrENz3Kie","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhYeysTWbJLkxwBrENz3Kie.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/hYeysTWbJLkxwBrENz3Kie.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"198b567fdfa37d6093d3fee4a4240ddf7327fd3e8789a4da0f10390af6f067a4","result":{"confidence_high":52.75236909834236,"confidence_low":33.24763090165764,"sample_count":100,"score":43.0,"score_display":"43.0","source_stated_rank":19},"run_fingerprint":"e4a73c1740442316135f086646648e847e8e1b16a9f73dec3e8e1bcb141494f0","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1892c2d81e4fa6351c9c31b145709419f8ba0920f84b71687e7566f7b6f42529","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.4278","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash (High)","source_effort":null,"source_model_version":"gemini-3.5-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3194eb015bdbed7128bfaecdd95ce7a9afe9ccb8aada513b5e6774c14e91a543","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.5,"score_display":"92.5","source_stated_rank":54},"run_fingerprint":"c98985309b21578e38fe5b2d9b71d6698efd30b539cd424f162f817369a4ca0f","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0e94d3df9f87a936c652bcee8d6543d23795c42071686ac5f79b9cdebe37bb48","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.0645","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash (Minimal)","source_effort":null,"source_model_version":"gemini-3.5-flash_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"81100669a27c647c724750021967dfd8db62cff75b81d261a9bc3b4b6312ed72","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.83,"score_display":"48.8","source_stated_rank":168},"run_fingerprint":"40531e44085c472ac025f07c1cf3efe8756b306f98646959ccc5577e180efebc","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_1393c364551ae24e70cd124e","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"794e463c7b788ced8ac879fb3fff4e7e95b19cd8b455568899144bb0c89e92b1","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.953882,"date_is_proxy":true,"latency_seconds":254.129,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.83},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"635108c364d9127c83662d7c594bfb752c9d1b603d934090ed2f4ffc2312ac07","result":{"confidence_high":82.3868,"confidence_low":75.2132,"sample_count":500,"score":78.8,"score_display":"78.8","source_stated_rank":28},"run_fingerprint":"f89b458069ede3afa6837d7740a088cec661deefd2c8cdf4a8a4ea1337ac3817","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":126,"at":"2026-06-01T10:22:55.131Z","basis":"evaluation_started_at","evaluated_at":"2026-06-01T10:22:55.131Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"40c9acbabaaf0f9b8eb69ca43fc091a54b32f5a29828cae089517f57df6f153b","metric_details":{"best_score_across_scorers":"0.7933884297520661","model_release_date":"2026-05-19","stderr":1.8422420250424887},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dwRzFGTghYEy4HNZajJXoc","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdwRzFGTghYEy4HNZajJXoc.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/dwRzFGTghYEy4HNZajJXoc.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"1b1d398accec0afee246bd3c6325d34434c11f62adf308065f9a8ec08f502601","result":{"confidence_high":82.9496373442899,"confidence_low":75.72804860612334,"sample_count":484,"score":79.33884297520662,"score_display":"79.3","source_stated_rank":3},"run_fingerprint":"62c1390a9608ab3b6235cd7ed91af8a994df77aeb905b3ae7c6df4f4ae90673f","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"c2d9fbd00900a879634dead4ec872148747718dc01090f62b1e14637ad6879ca","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-05-19","pass_at_5":61.81818181818181,"potential_contamination":true,"sem":0.9791208740244552},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","potential_contamination"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"Gemini 3.5 Flash__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"87cae4fc720c39d5f79492dbc012e5444b315f8a6cb11bd00d5822d440b577ef","result":{"confidence_high":51.37362236763339,"confidence_low":47.53546854145752,"sample_count":null,"score":49.45454545454545,"score_display":"49.45","source_stated_rank":null},"run_fingerprint":"aa675fde7b44f1357ce0d43de8b18ac3ed435f64c51df88a50a561f55ca43927","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_336dda2b1636ac9292005a45","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"1b64508ae33f3005a1dfcc82f19a4a06edcd10cd5aad07c7a75af2b592b07ee1","metric_details":{"ci_attempted":452,"ci_half_points":3.9663030105204387,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":163,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":105.30376940133037,"mean_cache_tokens":6428494.862527716,"mean_cost_usd":3.4467114588691796,"mean_duration_seconds":1125.1601607057523,"mean_input_tokens":7629071.858093127,"mean_output_tokens":75730.19290465632,"median_agent_steps":97.0,"median_cost_usd":2.8624087500000015,"median_duration_seconds":973.6770345,"median_input_tokens":5694082.0,"median_output_tokens":68910.0,"median_output_tokens_to_pass":64913.0,"median_peak_context_tokens":113979.0,"n_attempted":452,"n_passed":163,"n_tasks_attempted":113,"n_tasks_passed_any":72,"pass_at_4_points":63.716814159292035,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gemini_3_5_flash_high","source_model_version":"gemini-3-5-flash","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"e47ead2f51c572dbfc57ebe743690c269e3b2a60004b56d6c13a0123d91e6db9","result":{"confidence_high":40.02824991317531,"confidence_low":32.095643892134426,"sample_count":452,"score":36.06194690265487,"score_display":"36.1","source_stated_rank":61},"run_fingerprint":"35ff8572ac1eeb0e063ed04b1ee19ed55c00e59a9a51fca597972eee7b8986c1","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"025c75f24cd4bbe9128f5604dff0f304942f3d7c17dd7e1d0dc25dec88aede13","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"105.30376940133037","mean_cost_usd":"3.4467114588691796","mean_output_tokens":"75730.19290465632","model_release_date":"2026-05-19","notes":null,"pass_4":"0.6371681415929203","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-5-flash (high)","source_effort":"high","source_model_version":"gemini-3.5-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"f98696274ca75d1cf53fb378a1192b2c39de3384deefd83341fb2474683282e8","result":{"confidence_high":40.02824991317531,"confidence_low":32.095643892134426,"sample_count":113,"score":36.06194690265487,"score_display":"36.1","source_stated_rank":60},"run_fingerprint":"3a410283f1614a1d2db5b35f719386272ab07c0b02499d877823c73be9d1fd76","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"780db46872f7fa91630abb50e5e9d70ae96b617f60fcde3a7c50da8c85170cb0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"85.71681415929204","mean_cost_usd":"7.341823985508849","mean_output_tokens":"275777.50884955755","model_release_date":"2026-05-19","notes":null,"pass_4":"0.6637168141592921","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-5-flash (medium)","source_effort":"medium","source_model_version":"gemini-3.5-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"b61d1380c97f477429a9a6131be553ef58166e7345f4607a269ee46fafa06e8f","result":{"confidence_high":39.17724303775777,"confidence_low":35.60151802418914,"sample_count":113,"score":37.389380530973455,"score_display":"37.4","source_stated_rank":58},"run_fingerprint":"2f73bd72225ea3dc0f6dd213292b4b231992e048814344522f59155185a561e5","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_667b754faded84c3180149c6","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_50545b611e994b43926cee37","split_or_window":"google-single-attempt-comparison","unit":"percent","version":"SWE-bench Pro · Google comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":139,"at":"2026-05-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"41e2cbe9b0206872b766205a1ba0383c4ddac0e64a721a591de3bef865951de7","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · SWE-Bench Pro (Public)"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google DeepMind provider comparison; single attempt; exact scaffold, task revision and thinking level not disclosed","reasoning_effort":"reported","result_published_at":"2026-05-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Google comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"f68c03b01f7f9834b7acc612c405b63b4e486116c2485c289edc794e6ed13fb9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.1,"score_display":"55.1","source_stated_rank":null},"run_fingerprint":"6030db3d683f29b41168549aaeaac0cbb1f4904dc5313371f1c6ecd399d57e30","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · SWE-Bench Pro (Public)","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e45acdfcfb596c07e0973eed","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"b107c30c3daa98d438955ed1f003190d2d9c60f4a6b914624621055fb19d6755","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.94","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-05-19","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"79","tokens_per_task":"35105","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash","source_effort":"","source_model_version":"gemini-3.5-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"cfc3c5686084be142177b30978143b1d283a2cf941aad4a5c58aa728cfe5a93d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.8,"score_display":"49.8","source_stated_rank":61},"run_fingerprint":"b251a9e7e535af3f0ca84aea53309ea740545324571a3534323e7d16e7a7b469","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b6d1522f0f3f7acf9d6a364b","provider_config":{"source_id":"epoch-cursorbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8d4c059d0dfcef3230a6e1ee8bcc27cab8de99d1629210f7c621aad9b30cf737","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.085937,"date_is_proxy":true,"latency_seconds":64.371,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.95},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"2ba729f94c9446224b19a099f8d9ecfe91791bf11f63dc1fa5d3b50766b36e83","result":{"confidence_high":89.466,"confidence_low":85.742,"sample_count":null,"score":87.604,"score_display":"87.6","source_stated_rank":12},"run_fingerprint":"22a13a9d03ced555b1eca0a433402520a0a3491f1eed2b36d05ebbcad4af44cc","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b22699b0666fd04635d1ed03af5233c95b3148de629a039d4a159d25914ffa5f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash (high)","source_effort":null,"source_model_version":"gemini-3.5-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"9261aa3662ea71c91a5594798f8e66ec7b5fd9d208d6451b8790f0042fdf2e08","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.125,"score_display":"53.1","source_stated_rank":68},"run_fingerprint":"ded77f7ed9ff9fa1d11deee8ba0f6c24e03cf92dcf24393e5209c06e5e452d8f","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d66b85323ff2e09d05566279732028460d478c7c46108b652c8b63571bd55354","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.441921","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3.5-flash-high","source_effort":null,"source_model_version":"gemini-3.5-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"ff7c4e8d844244d6df0ad98ead4952ab0d5bf0cd9dd1e645c58dab28adfee8d8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":911.02,"score_display":"911.02","source_stated_rank":57},"run_fingerprint":"290f0bac74298d11151063c22b47543e78de1106bf8b3c807ccf4448e5d0929b","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4051d68f91cac1e69eb2a4d2a09538867b2b823ccbe626c2d84dabd0bed433ca","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.540385,"date_is_proxy":true,"latency_seconds":893.074,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.727},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"85ef1389edd1f874587e2be2236e74cbef184129819c7b301f61fdef43618b28","result":{"confidence_high":57.947919999999996,"confidence_low":39.41808,"sample_count":null,"score":48.683,"score_display":"48.7","source_stated_rank":53},"run_fingerprint":"25bc26e33ba50539df5d26363d15866a55e44e03e77d20a0446bb42a0522c55c","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9902d9dd8c94686e30fb86c9528f973b578f530ea3d9ac71136f7c9c67e9bd4b","metric_details":{"license":"Proprietary","variance":13.651758798297102},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3.5-flash-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"56ec755dbfdf53e841d01aab816b15469803f8679e9eaedfb8e248e53784a528","result":{"confidence_high":1506.2834259552615,"confidence_low":1491.7999637650657,"sample_count":9497,"score":1499.0416948601635,"score_display":"1499","source_stated_rank":54},"run_fingerprint":"e7feb5aaf5019f9d96bf4ee588b23749c5152663bfe08d494c47ad29dfdcb0fd","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e7d11ec5096290ef216e01af34b2ed8aa867b48fee11a76fb939ec30ff6c7030","metric_details":{"license":"Proprietary","variance":11.488814898286734},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3.5-flash-medium","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8deae54fde208870dd76f57beb0ac4b7b72ef8ad66e26219376719cb4d6d03d2","result":{"confidence_high":1496.6102437708525,"confidence_low":1483.3235874073657,"sample_count":12256,"score":1489.9669155891092,"score_display":"1490","source_stated_rank":56},"run_fingerprint":"6466dbf2bd45b430e5add93d87db0614ac1831665986c9b09de0642d002acff1","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_667b754faded84c3180149c6","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"57cfe991b830396c5a9a3d2c3d4f8302132aeeca5b90412799ce4f588527a1b0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.5-flash_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"029275d8cbf14dfe519ccc0d960f577e3002c83769e78f8058daa3e7f4b16979","result":{"confidence_high":1507.47,"confidence_low":1492.6,"sample_count":8931,"score":1500.04,"score_display":"1500.04","source_stated_rank":52},"run_fingerprint":"ab2d9da1f0e89df04cbe6471b597b48e07298584123a3278b9bf5d2a6ca1bf7c","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"18f1cd9d915ca7815fa4bbb4b02b21bbee497e101888e1b95d854e024acdb907","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.5-flash_medium","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"02f06f50cb9dc367fb9397953e880ffa23c8f73953e2de776af35e8331bb4c5f","result":{"confidence_high":1498.41,"confidence_low":1484.7,"sample_count":11412,"score":1491.55,"score_display":"1491.55","source_stated_rank":54},"run_fingerprint":"9fbea4eb2d8e5b44a9eb2c9341ed3f36f1d50268c5b5a0746c4a9741527a8303","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_667b754faded84c3180149c6","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a364f163a3ed3f280f7dc1101157dd2828025bf44f55807282d9dbc2832c2e7d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.5-flash_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"073cd093595c16485c16063f42fed391f14953cd3d8a84c2457f8a9b6f30c1bf","result":{"confidence_high":1519.85,"confidence_low":1492.91,"sample_count":2213,"score":1506.38,"score_display":"1506.38","source_stated_rank":51},"run_fingerprint":"1a2cb301a1a23fda238a670b01c042607c0f3fbaeee1a0ef7cb0890dc5c9aca9","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c9d6080777436d45ed0415fe","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d3d0706761fa883426e2e043ec1466fe93eb57a0a8127eeb69ac810de8114aa6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.5-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"6314b3888394b0ce27cf106727cd174d83c0e0776ca36ea799a0535697dd4c89","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.64000000000001,"score_display":"62.6","source_stated_rank":45},"run_fingerprint":"3dbba70607c1e5c366253075742598fdf1de64b4124aebf6804cd44fdc6b931b","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a55e3870150c7be930a8bc2b1ab38c26982cb64fc1f407cd94274c9463be7794","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.868179,"date_is_proxy":true,"latency_seconds":373.223,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.124},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"8d4262d625d8e1cc7a4246cc5aa91f395419c4321ff4d1f7246f053be9936404","result":{"confidence_high":76.36004,"confidence_low":71.95396,"sample_count":null,"score":74.157,"score_display":"74.2","source_stated_rank":20},"run_fingerprint":"df33aa107680181bc532fda3a5dbb8f375f800210119445ebbd0684de2399350","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_bde217160cbff9a7aeb3cce7","split_or_window":"terminus-2","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":139,"at":"2026-05-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"f033e3189eda1fdf34671bfc63008cb5250ca313d842bcea3aaa97c96e01f2e4","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · Terminal-Bench 2.1"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider evaluation; pass@1; default sampling","reasoning_effort":"reported","result_published_at":"2026-05-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"bcb4b343d34449dcf9aaeead2c13a11446366374408351358d9bc234c9274cbf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.2,"score_display":"76.2","source_stated_rank":null},"run_fingerprint":"755e048ca13b4a9df0b7f480f6171adde38dbd5dd240a892e96b8effe39a7274","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · Terminal-Bench 2.1","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e45acdfcfb596c07e0973eed","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":138,"at":"2026-05-19T18:56:30Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19T18:56:30Z","status":"stale"},"measurement_id":"ad1c3c05cdb00d90c8a2c3e66a3ea618a9802e5c30f36dbec86d5eddbc4432d9","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.3,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=Gemini 3.5 Flash (high); createdAt=2026-05-19T18:56:30Z","source_row_created_at":"2026-05-19T18:56:30Z","task_pass_coverage_threshold_percent":75},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"Gemini 3.5 Flash (high)","source_reasoning_label":"high","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"031afe892394501548f8f47fdf591f30e7ef37a411ad3503dc268257ced86132","result":{"confidence_high":85.89999999999999,"confidence_low":81.3,"sample_count":1000,"score":83.6,"score_display":"83.6","source_stated_rank":2},"run_fingerprint":"49fc5347ccb53c2666c4b6d34abb1621a47eea14905529fe014e8ef3556c7c54","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=Gemini 3.5 Flash (high); createdAt=2026-05-19T18:56:30Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"8bceecd92221db77944ffda42e2ab85e21337caf3d2e23a688cb5a09e26a2599","metric_details":{"expected_trials_per_mode":261,"normalized_gain":12.1,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":7.1},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":56.7,"confidence_low":39.7,"sample_count":87,"score":48.2,"score_display":"48.2","source_stated_rank":14},"run_fingerprint":"8d8c29bcb5bf7f7aa3dc0d1cf8a5006f636be15f031cb5c09f584289e32a5962","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c2aed1feafbd96f260e04542","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"069bd2ebc1a9b5ef729acb9fc084093353f9ba4f3efacc2d23dce82cc10e6fa3","metric_details":{"expected_trials_per_mode":261,"normalized_gain":12.1,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":7.1},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":49.900000000000006,"confidence_low":32.3,"sample_count":87,"score":41.1,"score_display":"41.1","source_stated_rank":6},"run_fingerprint":"6e4aef586ecd0840fba3b85ccc926141ecdde9b985700c11d16a988121fef598","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c2aed1feafbd96f260e04542","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"63155d382bc4c14fab0b70ddafa59d27810fdf6093c941fabfc6d6898d93c8b9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.443","mean_standard_error":"4.9","model_release_date":"2026-05-19","notes":null,"standard_error_points":500.0,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash","source_effort":null,"source_model_version":"gemini-3.5-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"26db5a7b1e54df8b261f7b072d6b843edf2ddf0b2bb3e8298e40a71ce161e7d0","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":27.500000000000004,"score_display":"27.5","source_stated_rank":38},"run_fingerprint":"4c43ef8b4bbefe45a0e4885379ea61cbad63aa16f9bd1bc445f7f0107ad9305b","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_97f8426db315c9e742f4bc9d","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8b423373d63ef00ebd6eb20dac596f37220b5aa33f3b451fc324864b40c558f6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash","source_effort":null,"source_model_version":"gemini-3.5-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"aa8132967196758e9cafa32477dd5915441e1de5b9abbf10bab0fc900987ef2e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5396.418333333334,"score_display":"5396.42","source_stated_rank":24},"run_fingerprint":"7f364b1d6367ff680f48c31db528d2005d35d29fb22e989572dee7ea1a40a844","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_81b485f877e695032b0db2d4","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"de1882f2f44a188fff4e59647d88e16542a3686439102dd6df285c1db6317465","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-05-19","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Gemini 3.5 Flash (High) · gdpval"},"model":{"id":"google/gemini-3-5-flash","name":"Gemini 3.5 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-5-flash","upstream_model_label":"Gemini 3.5 Flash (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"3cffbab8b4298f0fbfedd3d789495858459a09647b985dcac06a7bc5197c1a12","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1184.8,"score_display":"1185","source_stated_rank":null},"run_fingerprint":"93437758a10ffed0096fe53e603699b7edac3ae9623f0c819423c4dbfc91d0c6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eccca673c5e71c05cc78a977","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3b574913e1db6ef1c7fb640b50895dac0883d5d51f00205ec32086ba77e818e4","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-21"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 3.6 Flash; model release: 2026-07-21","source_accessibility":"API access","source_model_name":"Gemini 3.6 Flash","source_model_release_date":"2026-07-21","source_model_versions":["gemini-3.6-flash_high","gemini-3.6-flash_minimal","gemini-3.6-flash_unknown"],"source_reasoning_efforts":["minimal","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"63df3e2cc694208da456d1c0d5c63f2ccbf635c5ff553aa0f89588c991b3b6fe","result":{"confidence_high":156.28,"confidence_low":152.62,"sample_count":null,"score":154.32,"score_display":"154.32","source_stated_rank":34},"run_fingerprint":"95dbbf962717b8a41a02419812f1e6a0a5c68d4ef48ac04fb9feadf8152927d2","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_873f73507674253f1bc4bf8b","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c787913e193cb56ec6fd694b82651873f30c6731f8cd4c519dff1d7a3c36b97f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.6 Flash (high) · intelligenceIndex"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-6-flash","upstream_model_label":"Gemini 3.6 Flash (high)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"9f434d67b00ea2cd0e498d89fdd8980aa11a639b9950b8418aaa97b86c235aad","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.339568,"score_display":"34.3","source_stated_rank":null},"run_fingerprint":"ccf57ae381c0061119b32f1eff20d99f682f83fc734a8c0fd86f33b895ff695f","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.6 Flash (high) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef106db7e4e6da377abb2c7ff7c2ea5fb48c9772cfc0607de5c0308fd38ea374","metric_details":{"license":"Proprietary","variance":5.102522244354022},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1487.4396155306613,"confidence_low":1478.5849830519276,"sample_count":36070,"score":1483.0122992912945,"score_display":"1483","source_stated_rank":22},"run_fingerprint":"90fc649e6a175a7e9d04b53c71f28ab1391a15eac8effe6f8454fe5b12c2c38e","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f40d01097b1cbdb2addd14ca62d40ee0ee4a346294fc6eaefe1c0134059e6861","metric_details":{"category_scores":{"Agentic Coding":43.434333333333335,"Coding":77.863,"Data Analysis":62.99933333333333,"IF":75.36675,"Language":83.89766666666667,"Mathematics":86.4025,"Reasoning":85.149}},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.5875119047619,"score_display":"73.59","source_stated_rank":44},"run_fingerprint":"1912511055243c0118c5411b4ab41f3f628d11c6a0c00eef799dc3eee678af27","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f81463266f75b9a561c7e4c99e80f73cb5a270820c473f823fc615163379c12a","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.6 Flash (high) · hle"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-6-flash","upstream_model_label":"Gemini 3.6 Flash (high)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d3ce1e90a3ea85678481bfab2cdb77c323a79c154bc3d4580427b79aededbf6a","result":{"confidence_high":42.913141,"confidence_low":38.769143,"sample_count":2158,"score":40.824838,"score_display":"40.8","source_stated_rank":null},"run_fingerprint":"ee5de3c7354b1d9f8132e037f61f29b0e3ab40ebe7efec31f6c0068cf9867c7d","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.6 Flash (high) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:30:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:30:09.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c46cf5c802b222c181454148ed665e93134f2cd4be97d82b3d2d330c2b25a4a3","metric_details":{"best_score_across_scorers":"0.662","model_release_date":"2026-07-21","stderr":1.4965960710224384},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"F83pa7Gfsb4UNVttTnS5Hk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FF83pa7Gfsb4UNVttTnS5Hk.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/F83pa7Gfsb4UNVttTnS5Hk.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"0d6ad28d9fc396e46a42355e03f980b920d813e637e068660351d9f66798b9c6","result":{"confidence_high":69.13332829920398,"confidence_low":63.266671700796024,"sample_count":1000,"score":66.2,"score_display":"66.2","source_stated_rank":11},"run_fingerprint":"3ad4727af1c976357521c883db00b8cb1f4e7f6e4e4361e51ca6f4917edeb16e","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"26840c1f518b55fad3c8838940eea39c0d2b43f2084a4643a17496cd5eb80b50","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.03189,"date_is_proxy":true,"latency_seconds":18.064,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.329},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":92.424,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.6-flash","zero_shot_accuracy":94.444},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"8db929a8cab55d12448351b986c63862bbc9f28da882aaa01908b3c49fcb476b","result":{"confidence_high":96.03884,"confidence_low":90.82916,"sample_count":396,"score":93.434,"score_display":"93.43","source_stated_rank":7},"run_fingerprint":"0b6574ded7c9d4be7995fc28073eb5aef65b01f7c90aa1b009bf6bbb7c54f29c","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7cbcf2990a260276442ecb869733291d826457bd7476fee1d7666317783b8945","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.6 Flash (high) · gpqa"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-6-flash","upstream_model_label":"Gemini 3.6 Flash (high)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"494e9ed4daec1804bd1863b389714def1b2a08551ad6007cc2c14e32eac41c85","result":{"confidence_high":95.664893,"confidence_low":88.361393,"sample_count":198,"score":92.828283,"score_display":"92.8","source_stated_rank":null},"run_fingerprint":"0b0f904cdb794bc1a3a33e104a75dbeec8786e70922bcecb3dba83c4e47c03a7","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.6 Flash (high) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":64,"at":"2026-08-02T00:21:27.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T00:21:27.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aebf075a925ecdc87b56168031d3619eb9a544fa77e1f4788a13041a5dcc10e1","metric_details":{"best_score_across_scorers":"0.9412878787878788","model_release_date":"2026-07-21","stderr":1.4028967942064967},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"86SY5JbPB49xXhHFQW6tHu","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"4e8a2b5fedf532b2055e402573b9d340e683fe9b46819c1e2f789cbf1dd6722d","result":{"confidence_high":96.87846559543262,"confidence_low":91.37911016214314,"sample_count":null,"score":94.12878787878788,"score_display":"94.1","source_stated_rank":9},"run_fingerprint":"28d6e4cd3c5b33d675b82c65757fa4fa717cdb160dccdb779fb6bf7ffbc6e941","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:36:03.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:36:03.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8d4b3691d808d21b8dd935d50a1dda624de014964123c9e61cc1bea9469c95aa","metric_details":{"best_score_across_scorers":"0.8636363636363636","model_release_date":"2026-07-21","stderr":2.445015597318985},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"JoraLb9y55QXpMg4Rrmr5x","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJoraLb9y55QXpMg4Rrmr5x.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/JoraLb9y55QXpMg4Rrmr5x.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"65481583e9b3482ffac504fad1af25ed45b3ccc11fc0404e820e01900111cecf","result":{"confidence_high":91.15586693438158,"confidence_low":81.57140579289114,"sample_count":null,"score":86.36363636363636,"score_display":"86.4","source_stated_rank":76},"run_fingerprint":"1bc90090c632c6f368b6572ac072b43a05eb025bb28fb58b242b7143add4c19b","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0f80fd590fa845958d503e5e","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:17:01.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:17:01.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5d5e00058190001b7ebf8cb862138c179d1a37f9c852376f46b3db2345da8ab0","metric_details":{"best_score_across_scorers":"0.8585858585858586","model_release_date":"2026-07-21","stderr":2.4825909793343355},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"aPmR8fV49N98a6NVKpASAf","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaPmR8fV49N98a6NVKpASAf.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/aPmR8fV49N98a6NVKpASAf.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e90598eb2f3eda49c2d57b21b9e8202c75ce3bcd3a8ac832748b032fbe09fb98","result":{"confidence_high":90.72446417808115,"confidence_low":80.99270753909056,"sample_count":null,"score":85.85858585858585,"score_display":"85.9","source_stated_rank":82},"run_fingerprint":"e2aa7118d2ca6b7dba5169467b8db24e2b357d717871adf565773b8275dc411d","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_714e54eed00e9ff5a784cf1a","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-22T22:34:01.602615Z","basis":"evaluation_at","evaluated_at":"2026-07-22T22:34:01.602615Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cb4c58180d86533be85ed29d1ffe5fe14e3e6571b1a6aafac83e6eba10c5ca4b","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.6-flash","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bbdb3437aa9c40c41a5e5e5a7249e086e0e6383e867af14f724e425345385d86","result":{"confidence_high":91.1436621878251,"confidence_low":81.58361053944762,"sample_count":null,"score":86.36363636363636,"score_display":"86.4","source_stated_rank":11},"run_fingerprint":"c35392e8a0552583cddbf794f049154ba3c87e5c71bd99681bc09486c5857322","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_69c80e6dc2da2e076bfd636c","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2bd88007e3aff1b84f18696982498e0529a308290c71052107cfd475c91107e1","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.013327,"date_is_proxy":true,"latency_seconds":7.298,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.305},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65536,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.6-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c65ccf827751626cd0e2f972fabff969f1b42aa9386d5c3278d8708fc08afed9","result":{"confidence_high":89.8738,"confidence_low":88.67819999999999,"sample_count":null,"score":89.276,"score_display":"89.3","source_stated_rank":13},"run_fingerprint":"8c9922ddd2885cdcd1547cac8a95652b5664a456f42b7518fb953d6035742f21","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T00:21:27.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T00:21:27.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2dd4d896048c3df01bf569873bbbd58da0f8c585ccb5f94477b31a07700b51f4","metric_details":{"best_score_across_scorers":"0.9416666666666667","model_release_date":"2026-07-21","stderr":3.092635435901545},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ajQFZGMjvyMTH2ULDZhwtz","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4f13d3d4ab35bb573e108a1e14ddd9d6029fdc2f9388cfd160170901439a0866","result":{"confidence_high":100.0,"confidence_low":88.10510121229964,"sample_count":45,"score":94.16666666666667,"score_display":"94.2","source_stated_rank":50},"run_fingerprint":"1df0ee075577f0d24cbf4ddd10a472eefa7418bf29d08c248f453c7c5e87f5bf","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:07:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:07:20.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b3e3ef6ddfcf2d90b8e829ec59292c15090cc1882b4e9cc805c19554200ed7dd","metric_details":{"best_score_across_scorers":"0.8222222222222222","model_release_date":"2026-07-21","stderr":5.763774795025092},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6iEkMa88YjLDgcde4bLm8B","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6iEkMa88YjLDgcde4bLm8B.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/6iEkMa88YjLDgcde4bLm8B.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f2ef320053cdb001740339d14b4c855f1fb0207967030fb6863c2b98d7ed0462","result":{"confidence_high":93.51922082047139,"confidence_low":70.92522362397304,"sample_count":45,"score":82.22222222222221,"score_display":"82.2","source_stated_rank":103},"run_fingerprint":"52d3e29dcfc87346bd23344d0edc6de2e9bc2e2a6e3b82645e1b9c8b490c818d","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0f80fd590fa845958d503e5e","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:07:32.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:07:32.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b7cb9469797a8454ae9e6ca6b6cf6f3e9219695160237099cc548bc44961d770","metric_details":{"best_score_across_scorers":"0.8","model_release_date":"2026-07-21","stderr":6.030226891555273},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"cF97JqHbFxxp7ePjYVK9HV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcF97JqHbFxxp7ePjYVK9HV.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/cF97JqHbFxxp7ePjYVK9HV.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ee2dd4fc3c3526a2d29b7b97758eb8c6b84e3ae83a025781d4776ea9589d59c3","result":{"confidence_high":91.81924470744833,"confidence_low":68.18075529255167,"sample_count":45,"score":80.0,"score_display":"80.0","source_stated_rank":110},"run_fingerprint":"4c7bbbcb57cdb26a759ea93c2249ad33b3a4b3d409a125d102dc48c8d9f06dd7","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_714e54eed00e9ff5a784cf1a","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T00:36:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T00:36:45.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e622165a18667927664c7d7ec844d8a027bc443b51e1473e8703f62ffe9dea81","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5894736842105263","leaderboard_private_problem_count":285,"model_release_date":"2026-07-21","public_example_count":10,"stderr":2.9190634943914042},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"nb9LnDeeQyvRCqsbsTANGD","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fnb9LnDeeQyvRCqsbsTANGD.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/nb9LnDeeQyvRCqsbsTANGD.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.6 Flash","thinking":null,"upstream_model_id":"gemini-3.6-flash_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"7009711d176684ec2286b418347a8c55d59be076e6ce742303dbfab48fa05624","result":{"confidence_high":64.66873287005977,"confidence_low":53.22600397204547,"sample_count":285,"score":58.94736842105262,"score_display":"58.9","source_stated_rank":37},"run_fingerprint":"c96ca9b1bda76dd01bfcacd2a4524d5a0463faaaa004d60dc4cc27ed39f0cde8","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"84cccfcd24c2a217a3c8de8421c9cd6876959f27ed818756cb62f66a756c01ff","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.6060796625000002,"model_release_date":"2026-07-21T00:00:00.000Z","results_url":"https://arcprize.org/results/gemini-3-6-flash"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-6-flash","model_type":"CoT","upstream_model_id":"gemini-3-6-flash-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"554223a6af1805cd60039a3dbf8b91eca97034772aae1bb886053a411bb9daf0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.416666666666664,"score_display":"60","source_stated_rank":76},"run_fingerprint":"20fe99732d49c380147b44ad4647325b07e95fffa35d690fa18b7858fe668f22","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0717860b8c60372e5c8079d2caeedd4795707e514e4ca7e1f2ef3fa15213625e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.45550909999999994,"model_release_date":"2026-07-21T00:00:00.000Z","results_url":"https://arcprize.org/results/gemini-3-6-flash"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-6-flash","model_type":"CoT","upstream_model_id":"gemini-3-6-flash-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1e4b291a9c5955a4d6d6479b0c511da43de250317e26dfb1395de623e8b259c4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.416666666666664,"score_display":"50","source_stated_rank":95},"run_fingerprint":"69f72f7f52dde34c662ae8e6305c254795d899abe6f955f16a99c647a85178b4","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d6f7288969ac5442efd6b4b1","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"03d4e841b39e9c6eceb82ba862268430e80ed9d03f783843b3c4d8b53781fb7c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.22297816250000002,"model_release_date":"2026-07-21T00:00:00.000Z","results_url":"https://arcprize.org/results/gemini-3-6-flash"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-6-flash","model_type":"CoT","upstream_model_id":"gemini-3-6-flash-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"97dad32061e856f8d0eb8d22cc72ad019d266ee2e3acf54f2856c6e0239135f0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.416666666666664,"score_display":"30","source_stated_rank":118},"run_fingerprint":"60670bdba046fe7fd90e02332d7cbf3c0e2e46de8f1bbcf48ad1d4c7f5db8da4","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0f80fd590fa845958d503e5e","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"235bcdbae6dc3690ee86db585c571f0ad60d9edac0b6d9b7249d80e6e35e47c5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.07084053750000002,"model_release_date":"2026-07-21T00:00:00.000Z","results_url":"https://arcprize.org/results/gemini-3-6-flash"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-6-flash","model_type":"CoT","upstream_model_id":"gemini-3-6-flash-minimal"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1547b5e42f0b24af2ca04393d6c51d378e20694150c87032e82d54a54ed25354","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2.638888888888889,"score_display":"3","source_stated_rank":186},"run_fingerprint":"e530eac20651c4f40446299b42a45cf30c110d7cfd26b9677ce4b658d5c891c5","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_714e54eed00e9ff5a784cf1a","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9c4952175c53eab82e6f3b7025b9d76ccc191cb509645f31a159addc098989a3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (High)","source_effort":null,"source_model_version":"gemini-3.6-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c2fafbe1a16a81e20ba4ad9031490a85007df3eb7bf248cfa0cc706decfcd9e6","result":{"confidence_high":65.332868,"confidence_low":55.280498,"sample_count":360,"score":60.416666666666664,"score_display":"60.4","source_stated_rank":79},"run_fingerprint":"8139e754ed6d516da2efeb30c4cce8a766b97d36560d35a7159ba05dd4af588f","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"406988535cc29000f2f116026fe2a13a7f7a2a046dbbf068ab4d0dadbad5be99","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (Medium)","source_effort":null,"source_model_version":"gemini-3.6-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"71df76ac0e10ee5a8d51b5f19bcba87c01b47baf77483f6c69ec1e5d63e028b6","result":{"confidence_high":55.549805,"confidence_low":45.27473,"sample_count":360,"score":50.416666666666664,"score_display":"50.4","source_stated_rank":98},"run_fingerprint":"8c48f6f5f19a8507cbe2fbc6162fa3ffee0d1bc2ded604935666f9da8c937150","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d6f7288969ac5442efd6b4b1","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8d8480263b4a50c7115e19630b2a0db480c46e7c443e07fd7f9e7d23c57db37f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (Low)","source_effort":null,"source_model_version":"gemini-3.6-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"7331ec62376c6b22873487daf757241ec7b422cfcc8f0d028498cec2b83e06f8","result":{"confidence_high":35.355203,"confidence_low":25.891669,"sample_count":360,"score":30.416666666666664,"score_display":"30.4","source_stated_rank":118},"run_fingerprint":"a92aa19722a2cea3ff41f14e3fef336abd64a4053f9ad563fdaa6f06f9971073","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0f80fd590fa845958d503e5e","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ec101fc2ccfab3251320012d207ef7ec226da21f8e550333ed7b1f1175818dea","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (Minimal)","source_effort":null,"source_model_version":"gemini-3.6-flash_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"549627c9837b9632b43dc3fd0ec2fccab0e1988d65a16fa62ffa79a9cba5bcbd","result":{"confidence_high":4.860224,"confidence_low":1.417673,"sample_count":360,"score":2.638888888888889,"score_display":"2.6","source_stated_rank":181},"run_fingerprint":"dc973793ea0d3c8d8832e21d604296c99ec2cc1b7d1edeffca10e7d1ceecc0dd","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_714e54eed00e9ff5a784cf1a","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T00:36:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T00:36:45.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7bfcccf7d039412054eb0d1edd68b633d936adc178b3c676fd201a716d7495a6","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.21951219512195122","leaderboard_private_problem_count":41,"model_release_date":"2026-07-21","public_example_count":2,"stderr":6.5445892024384085},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"JCMjXU6FRiiLV3xvs9R2Xv","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJCMjXU6FRiiLV3xvs9R2Xv.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/JCMjXU6FRiiLV3xvs9R2Xv.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.6 Flash","thinking":null,"upstream_model_id":"gemini-3.6-flash_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"7157a5167fbcc14c246da0a6790e6feb872cf05de840639b974c69b75ce336f0","result":{"confidence_high":34.7786143489744,"confidence_low":9.123824675415843,"sample_count":41,"score":21.951219512195124,"score_display":"22.0","source_stated_rank":44},"run_fingerprint":"646d1bb6df42090d2a1741277714aba66595e2793f395595a267df56cc1a1aec","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"87085ba2dfcaf133dce9b9e092e11bff9130d188d3f2b22756c64ad8b8f1835a","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.0186,"date_is_proxy":true,"latency_seconds":11.313,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.77},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65536,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.6-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1215745ed2a72b589100844a4005721a9d3bb0100050a0bb0313345b38919c5d","result":{"confidence_high":89.89120000000001,"confidence_low":86.8728,"sample_count":null,"score":88.382,"score_display":"88.4","source_stated_rank":7},"run_fingerprint":"e6771cbb977e75d6cdd00eacc3cf85b08f7de2f5f71d9ec18834b47e37691b34","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ef29b9d9f6ed2e3f9a7a046efff364bfa11e82ad8bcaed977d2a98e108ed7821","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.6 Flash (high) · mmmuPro"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-6-flash","upstream_model_label":"Gemini 3.6 Flash (high)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c585f7707a6a52d803759777c8e64fad690067631e5e61911582aa2de9e8d9fe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.236994,"score_display":"83.2","source_stated_rank":null},"run_fingerprint":"590a8081c7afbf243e9fd4ee678dd2951f1879e9ab9efbb3a2d21aaa2c1586c8","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.6 Flash (high) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c7d8e22564c96b6d5794cccc4e0e5e7bd9c2fd4dd5680efe7fb716ca10ced38e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (high)","source_effort":null,"source_model_version":"gemini-3.6-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"cbe94884fd12b808a92d4ddbbd1f9923cb827e396c839dac824bb0a3b0097b67","result":{"confidence_high":19.849425,"confidence_low":5.341149,"sample_count":71,"score":10.5714285714286,"score_display":"10.6","source_stated_rank":90},"run_fingerprint":"acf5623108d7f3d18340c24c66fc743e64cc7c22329c2cf0acce776eb1017560","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b70ef1f272564203ff826fe82a343eb70cca4b4e868daf028b34f8f5302228da","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.6 Flash (high) · critpt"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-6-flash","upstream_model_label":"Gemini 3.6 Flash (high)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"047ed72957f64e18c9a282e29d6985ed2ab984bc83903effe903a733aee3c448","result":{"confidence_high":19.929636,"confidence_low":5.315755,"sample_count":70,"score":10.571429,"score_display":"10.6","source_stated_rank":null},"run_fingerprint":"710331691f1df855f26d28d0f72a3dd37a5334849fd718f11b5d73908b43292d","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.6 Flash (high) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T00:21:27.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T00:21:27.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"258b43d94d55873e4bf85a4f0a810d75495812594603d89942152b9fdf154468","metric_details":{"best_score_across_scorers":"0.4","model_release_date":"2026-07-21","stderr":4.92365963917331},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"aMtSkxyLZPdf6o6f5zLP6A","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0c5a8085054aea781cba308aeaa171699ce624b3e0b3d234f714388ff1a4fc2d","result":{"confidence_high":49.650372892779686,"confidence_low":30.349627107220314,"sample_count":100,"score":40.0,"score_display":"40.0","source_stated_rank":24},"run_fingerprint":"933b148d03e9b70b8d6388b9a23d4d224356eeec6a3bfe3f06629b794f3c28d2","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:17:08.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:17:08.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3bbd0483a33a9e34db7a711a5439bc504ece872305a528705de60f29e354b4a5","metric_details":{"best_score_across_scorers":"0.43","model_release_date":"2026-07-21","stderr":4.97569851956243},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"DLvSYBHHLcHECeL5SpNgVK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDLvSYBHHLcHECeL5SpNgVK.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/DLvSYBHHLcHECeL5SpNgVK.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"422a7f9ae9b6ff470aa495978f68ac568f66faf5564a47fdcfc1d9603c711cfc","result":{"confidence_high":52.75236909834236,"confidence_low":33.24763090165764,"sample_count":100,"score":43.0,"score_display":"43.0","source_stated_rank":19},"run_fingerprint":"ebdb17f4b2cc0ba6669e1dddf52393d531b385990f07705a4c4a787dbbf92da5","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0f80fd590fa845958d503e5e","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:20:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:20:15.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f5353cef735773fbdfc0a256cf19ff2456a953712045eafdd823a6ec4b81edb7","metric_details":{"best_score_across_scorers":"0.35","model_release_date":"2026-07-21","stderr":4.793724854411022},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hL24cexf97QKVLsv73H5TT","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhL24cexf97QKVLsv73H5TT.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/hL24cexf97QKVLsv73H5TT.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"bca96109e32e31f20ac4b482017ca2820ccbd149095f6637a5a9d47342592c98","result":{"confidence_high":44.3957007146456,"confidence_low":25.604299285354397,"sample_count":100,"score":35.0,"score_display":"35.0","source_stated_rank":40},"run_fingerprint":"399adb36780dcbdcd3d95ba2aa935d3f13d56a8fea946c2f193b049c6d915754","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_714e54eed00e9ff5a784cf1a","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a23122bcfee0987af6956ba1d892cac7e349b73488eabf7bd389da946dbe2a83","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (High)","source_effort":null,"source_model_version":"gemini-3.6-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0ff0cecd15fa0dc7c17389bbe906b92d29f43f4496ab963d0a22a57c92b920bd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.16666666666666,"score_display":"91.2","source_stated_rank":65},"run_fingerprint":"60826e2303b57d01518ac274b2fc03f779ea170d11c6e5bf5f5ae2cbac7b62d5","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b72c859402cf077d727111b53fdcb30075b7ac486276400f7aeea46b3196d268","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (Medium)","source_effort":null,"source_model_version":"gemini-3.6-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"02dc2b3e356ab1147d6202f588dfe2030e4778fe726815386663453297b26cbe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.16666666666666,"score_display":"83.2","source_stated_rank":109},"run_fingerprint":"24586bea0895b551777302767900aaea1afcc4f80c54bea37d4fe4aab6878a4f","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d6f7288969ac5442efd6b4b1","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"54b60e929f44c692c7d81169284eba083853520c2cf46588edf7efafe1c0b55c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (Low)","source_effort":null,"source_model_version":"gemini-3.6-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"92b39ed9be48695bf4f5b9c7ae0df27559f526f6fbeb48bd54cca3092813ee13","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.5,"score_display":"76.5","source_stated_rank":119},"run_fingerprint":"d59cd6965fb656e75487b582adb6bc1db29363f9d6f9992f5dd5c248af182979","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0f80fd590fa845958d503e5e","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"88acd130583a76f747bb94dbb95e98c873491ea5ddc9f5d597d9bbd8e77389e5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (Minimal)","source_effort":null,"source_model_version":"gemini-3.6-flash_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a5fe95395efde2e7dc5a0721950ec5c9abd4479f724580e951d2667b07330d46","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.5,"score_display":"34.5","source_stated_rank":190},"run_fingerprint":"7f553cc79018bfac5049d36cca8845b1eda95259996475cc51e61396ea6ff046","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_714e54eed00e9ff5a784cf1a","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c24d4d294d54f6fc4cda38c4dbd51bf01c9379871ed43650fbba1b5da141be7b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.194601,"date_is_proxy":true,"latency_seconds":788.09,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.804},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.6-flash"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"1c7190a8847c304e8b7aa0e89baf7fb51832952f50d8560b57d05dd5d10d0fde","result":{"confidence_high":83.13583999999999,"confidence_low":76.06416,"sample_count":500,"score":79.6,"score_display":"79.6","source_stated_rank":25},"run_fingerprint":"53961d2978c6e7cfb07ddb0609e92ce5d2d04c9bcc02bf4f989aa3bc0bed57ca","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"5bbf6f0bb61d9ec86f5d65c6e22d3a4e43edbb202c5be6ca50bca6ccd65262bd","metric_details":{"ci_attempted":452,"ci_half_points":3.7048538992472775,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":211,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":116.73333333333333,"mean_cache_tokens":11254636.448888889,"mean_cost_usd":4.419013277333334,"mean_duration_seconds":1477.8607510707964,"mean_input_tokens":12595957.344444444,"mean_output_tokens":95844.86222222222,"median_agent_steps":108.0,"median_cost_usd":3.444695325000001,"median_duration_seconds":1214.8670550000002,"median_input_tokens":8949018.5,"median_output_tokens":86727.0,"median_output_tokens_to_pass":83414.0,"median_peak_context_tokens":151398.5,"n_attempted":452,"n_passed":211,"n_tasks_attempted":113,"n_tasks_passed_any":85,"pass_at_4_points":75.22123893805309,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_gemini_3_6_flash_high","source_model_version":"gemini-3-6-flash","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"808897a2ada0f4bd1c0907516eeb1e63004ffe30d1bd8bbfb33ef7e9bf5968c2","result":{"confidence_high":50.386269828450814,"confidence_low":42.97656202995626,"sample_count":452,"score":46.68141592920354,"score_display":"46.7","source_stated_rank":53},"run_fingerprint":"3c5c812d091e53f9b1392a08c0d38d56b4b55516ef8be2abafd11ff6c82052ae","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4833a834661daab5d7ceecda2d331bb4cf06f8ec23d7425978c241ce211983c6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"116.73333333333333","mean_cost_usd":"4.419013277333334","mean_output_tokens":"95844.86222222222","model_release_date":"2026-07-21","notes":null,"pass_4":"0.7522123893805309","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-6-flash (high)","source_effort":"high","source_model_version":"gemini-3.6-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"496256eaf307e9d7396426bd360c0ee95e355bab7373bae1e67f216bfd83676f","result":{"confidence_high":50.386269828450814,"confidence_low":42.97656202995626,"sample_count":113,"score":46.68141592920354,"score_display":"46.7","source_stated_rank":51},"run_fingerprint":"b2ab3a129d1397ef1c4c97abb9408d6ed7d623fd24f2aaf151cb87cc1c299fdf","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_bf1ea14003c917dab90422d7","split_or_window":"vendor-public-own-harness","unit":"percent","version":"SWE-bench Pro Public · Google provider run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":76,"at":"2026-07-21","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:32Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-21","status":"fresh"},"measurement_id":"cf10204ba3aa9351998f3e23e7cd6de1129504b5be53e664b25c02d4271f27cc","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/","evidence_sha256":"d148585d223f2bf4596bd5dca47ca518be4891525cd3e18a3eeb7599455505c4","kind":"reviewed-static-report","published_at":"2026-07-21","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/","version_id":"e2b9f9e2736a881433c558e4098f782716af51b1e54bb054f3f055aac8b3d45a"},"source_locator":"Model-card benchmark table · SWE-Bench Pro (Public) and evaluation methodology"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"pass@1; default sampling; Google self-computed Antigravity evaluation; exact thinking level and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-21","results_as_of":"2026-07","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"aa275c6d076871ccb0440ce95de82b94963f9d951e6a1772b068d42ff28607fb","source_published_at":"2026-07-21","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Google Antigravity evaluation infrastructure","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"448f907193a51ab3be38b2b8c6b2655f2f97c34f00868f29797d376c7d282b25","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.7,"score_display":"58.7","source_stated_rank":null},"run_fingerprint":"2d2080d7dae33a999935c3dc274b3a68d5ddb64c5b13d7ab08650a92afc1eba5","source":{"content_hash":"00b894325b1d599b95a6f00fa33353412d37818fa36735786b1cc1e024427117","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/","id":"google-gemini-3-6-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · SWE-Bench Pro (Public) and evaluation methodology","name":"Google DeepMind · Gemini 3.6 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ac0fb2fe1b81677fe3c51965","provider_config":{"source_id":"google-gemini-3-6-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"697f610cc8ba51825f59c242899a45eceda8caf23b17cf2887ccd85578fd87e5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.56","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-07-21","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"64","tokens_per_task":"30436","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash High","source_effort":"High","source_model_version":"gemini-3.6-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"40b0bce188fe4e66137c33896e9b8b4ad2271c311f7ea67a05d35eec77f33665","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.5,"score_display":"53.5","source_stated_rank":52},"run_fingerprint":"c5257da23af1663e84dae951b4e5cecce117b3c8822135a7efbc6e147774a01d","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"3cc3c2815101af2c09ef636c23a3245d35d71ea18f2cab676c457e47e0a335a7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.48","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-07-21","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"62","tokens_per_task":"28511","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash Medium","source_effort":"Medium","source_model_version":"gemini-3.6-flash_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"435f715792fbcbc8f2ee7269239fdbf65c19ab1221137e58a537186a46a89b28","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.2,"score_display":"51.2","source_stated_rank":58},"run_fingerprint":"235d0a190fe2805f91637b3d0a003b3af47618db51f64dcf9d09b6b630b32495","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d6f7288969ac5442efd6b4b1","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"f9bf2511d6cbd18f91e9d6650a57fd7039fead078e728f2cd149a1034e672bed","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.13","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-07-21","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"50","tokens_per_task":"20529","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash Low","source_effort":"Low","source_model_version":"gemini-3.6-flash_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"4ffc8f48a8beacd18b6fd876bcf7cfbd4ae5df1d611513f5792d2c2db957926f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.4,"score_display":"47.4","source_stated_rank":69},"run_fingerprint":"49781e6d9de584c2c1845d8c47657c017084ad7a8a8e04988345aee63fd1f832","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0f80fd590fa845958d503e5e","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"56b72f05b3cd7d97e54e54e9392f024395b757436fa2d03e800fd91ea9bde1d9","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash","source_effort":"medium","source_model_version":"gemini-3.6-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"chisel","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"665e34912f8fe7c57fe003d316d8a54a71e014c6dd8e4de7dfca9b8ee4a975db","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":34.37,"score_display":"34.4","source_stated_rank":25},"run_fingerprint":"892d6024aa0ea7bb04a3fb301ef1dd569863101de2e9994234f635a87e8be46d","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_d6f7288969ac5442efd6b4b1","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7f173d7856e97af89abd664ccd3065c3ea43e3fa4c8c419adb12b0928bf8836b","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":8.66611,"date_is_proxy":true,"latency_seconds":2903.307,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.695},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":65536,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"high","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.6-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"d6fdaa8a832acfeb642443210b718752ac67141678a9e632f3010535b495c3c2","result":{"confidence_high":46.218199999999996,"confidence_low":23.8938,"sample_count":null,"score":35.056,"score_display":"35.1","source_stated_rank":35},"run_fingerprint":"ae62a452181342e139713c8209ff50204d3578922175bc49d374cf10ae05b966","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c405b2b7e496ad564dea9d3f6ba1ff6a3535dbb5ab4942dd6ed912e151f89a74","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.056383,"date_is_proxy":true,"latency_seconds":31.883,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.939},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.6-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"85c31ee08dcb54051adadeba1509f3dfb9c17a484ca6ffdf4e6aeb2186ff4999","result":{"confidence_high":89.92044,"confidence_low":86.23956,"sample_count":null,"score":88.08,"score_display":"88.1","source_stated_rank":8},"run_fingerprint":"aa89548d236730452c8acc3f26c927abb11cd18a7647f8f3097df011ebdbc949","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"922d50314b0a9ce788f43d753788846ea90616b85b8e00cf442b2dfbd2fbe19c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash (high)","source_effort":null,"source_model_version":"gemini-3.6-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"a917ae069bf0c504a7b2274e0700c878daf09d69fc769368693fa163fd0e966d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.662037037037,"score_display":"52.7","source_stated_rank":71},"run_fingerprint":"653f0b1fdc31121684f70cacb4a5fb2ddaafc57e921f63cc6e7bc5a856c734fc","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e096768ed4baa7a2847e46ff53a22797ad90950e2889be1d8725e66a4125a54f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.6 Flash (high) · scicode"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-6-flash","upstream_model_label":"Gemini 3.6 Flash (high)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"82b0d3b30a3b7d3a1904f6660012b5a49582fa290546bbcc6f007d272e4cd27e","result":{"confidence_high":59.036101,"confidence_low":47.588496,"sample_count":288,"score":53.356481,"score_display":"53.4","source_stated_rank":null},"run_fingerprint":"fa7544d9b532639c900ad4279227769a4a14192353cc6367d3f8fb85b0acaf57","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.6 Flash (high) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"746d661fa36d2e92464d53a60fcb7df49afab16cd8932734a4e85e918edca2b8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.244468","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3.6-flash-high","source_effort":null,"source_model_version":"gemini-3.6-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"7cd0014ca862437fbdce8d22a81e5a4e5e55089d061b1b3238417a012545d34d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":715.52,"score_display":"715.52","source_stated_rank":78},"run_fingerprint":"b0a3d4125750cd06308667c6d18122ead7d57694c26044fd1c595549becd22dc","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fb4a9a947970f8d624360e3190d007679c137fed5b0c0717ffd45a1f1fd5ab36","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":3.041993,"date_is_proxy":true,"latency_seconds":1523.201,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.29},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.6-flash"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"826ea06a1eb0ae6f680931a22569881304011b4b82df2bed9f1de9e48bfec7e1","result":{"confidence_high":72.4134,"confidence_low":55.596599999999995,"sample_count":null,"score":64.005,"score_display":"64.0","source_stated_rank":42},"run_fingerprint":"08fc016e16e56712e3cafc546c3435b5fbda3445715dab234025dbde8910a941","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"92aaddff8c1d284a35a9b61dc98a9b58c146792c6aa64c95e5e00506c837e490","metric_details":{"license":"Proprietary","variance":14.4950410092205},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3.6-flash-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7d0cad77f18217de93d82d51f982237a8ce3f7dd846f36d9062ca2b0a3ba530d","result":{"confidence_high":1543.7331649547539,"confidence_low":1528.8090766219686,"sample_count":8823,"score":1536.2711207883613,"score_display":"1536","source_stated_rank":42},"run_fingerprint":"cfb2652b647d41f9c4b9e0d076f0b8744cd3e916d035f8440e7e754cb8647916","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"118ed9441c077e92149d7e83e07f481b3cce2108811821804e2c23149fa22797","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.6-flash_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"c908fa268c2414fc58182ed655557866f7e6dd1cdb66ff4a16e71efbbcebe22e","result":{"confidence_high":1544.62,"confidence_low":1529.28,"sample_count":8259,"score":1536.95,"score_display":"1536.95","source_stated_rank":39},"run_fingerprint":"75c3ff080ef5b8c318a39635f37ddc6c361f2b9a14ada51f06740f050dc670d2","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a5fada8303319cbf57928faa9e64cd75cf1e2abd1956bcdc034acf90b6306086","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.6-flash_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"d3d39b6446c1b4b1377eca0ba317a9f9d1f6f3467bce507ffb85ef2fd9361040","result":{"confidence_high":1540.33,"confidence_low":1515.33,"sample_count":2768,"score":1527.83,"score_display":"1527.83","source_stated_rank":42},"run_fingerprint":"fac5a082ecf01b16bf940242b22471f88997918ad27d77f608ba0c298e2fc709","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0b3004d5d838e134e0bba278","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2a2e758ce21ca45ace0e4aba18209951c73bd6dad094991e3920a7e9169b9b70","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.6-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"fed35aefbb9ecf5bdac93fe42a07a1389116668de3e39f8adb4adf526dfbe273","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.10000000000001,"score_display":"56.1","source_stated_rank":58},"run_fingerprint":"475f44969dbdbd3a92eb8932d9f953e2de7d336f37b681894929f63423769f02","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3a4e49fb9219a48845f802a97597bf0a358d2702d22d93e0d5e816cb2f899a11","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.174404,"date_is_proxy":true,"latency_seconds":552.153,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.633},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.6-flash"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"b8b3cdc72f67b5fcaaa3e892a607ab7006af03e14a8d1aa7f4713f76d0b4f85e","result":{"confidence_high":76.98368,"confidence_low":70.58232,"sample_count":null,"score":73.783,"score_display":"73.8","source_stated_rank":21},"run_fingerprint":"338c787d8f7fbaf441c6c0242cb40ad18ef53016276744e465b04ee2bafdb40c","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"752f1d631bbdd57c2b0ec45b754e130bcaeb552322d265af224f1f1484ff962c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.6 Flash (high) · terminalbenchV21"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-6-flash","upstream_model_label":"Gemini 3.6 Flash (high)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"94594acbae7c3deb0e7c374e47132a2436c09108497ae2e05fc9ddbb83700968","result":{"confidence_high":84.955618,"confidence_low":67.822447,"sample_count":89,"score":77.52809,"score_display":"77.5","source_stated_rank":null},"run_fingerprint":"1e5f41d39362d7c815e47ecdc60073b1bb3d43f942b6ea906dcdd1f1a88d9548","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.6 Flash (high) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_bde217160cbff9a7aeb3cce7","split_or_window":"terminus-2","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":76,"at":"2026-07-21","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:32Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-21","status":"fresh"},"measurement_id":"7936be7a672e8116e448fc29303b81bbac4984f1be30cee7bdeb8f51b3708184","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/","evidence_sha256":"d148585d223f2bf4596bd5dca47ca518be4891525cd3e18a3eeb7599455505c4","kind":"reviewed-static-report","published_at":"2026-07-21","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/","version_id":"e2b9f9e2736a881433c558e4098f782716af51b1e54bb054f3f055aac8b3d45a"},"source_locator":"Model-card benchmark table · Terminal-Bench 2.1"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider evaluation; pass@1; default sampling","reasoning_effort":"reported","result_published_at":"2026-07-21","results_as_of":"2026-07","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"aa275c6d076871ccb0440ce95de82b94963f9d951e6a1772b068d42ff28607fb","source_published_at":"2026-07-21","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"56601aca45b0e5b0f98dff5cc9afcefdf0ba33bfefa8068fe89b55d8708772d9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.0,"score_display":"78.0","source_stated_rank":null},"run_fingerprint":"67a8feaef700057855e6535d15776e7dd3ac5732df7fec4367c3f28e51d2a482","source":{"content_hash":"00b894325b1d599b95a6f00fa33353412d37818fa36735786b1cc1e024427117","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/","id":"google-gemini-3-6-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · Terminal-Bench 2.1","name":"Google DeepMind · Gemini 3.6 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-6-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ac0fb2fe1b81677fe3c51965","provider_config":{"source_id":"google-gemini-3-6-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e5228411478cf1ef1a5a818687b8d841e65f341e699e20bcfd149825c08e1553","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.6 Flash (high) · tauBanking"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-6-flash","upstream_model_label":"Gemini 3.6 Flash (high)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"512734a6a28523bf4572c6c83b3ed7007e69ec3a01528bf06a4fc4daa040feef","result":{"confidence_high":39.630982,"confidence_low":21.694503,"sample_count":97,"score":29.896907,"score_display":"29.9","source_stated_rank":null},"run_fingerprint":"5fcb31d401a9479d755bdace4f2c6770ea2bdac2faedb91bbe54400ce376ec64","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.6 Flash (high) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"00b3cc2c1a148336572abb3c06afd70b7be008f30de7c041cd620bf069e3534f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.634","mean_standard_error":"4.4","model_release_date":"2026-07-21","notes":null,"standard_error_points":520.0,"upstream_source_url":null},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.6 Flash","source_effort":null,"source_model_version":"gemini-3.6-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"a254d7d4134c37691c83cf79900b3b5f057d69be7906b813abe3715afdb370f8","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":46.9,"score_display":"46.9","source_stated_rank":27},"run_fingerprint":"f8abd74d91d276d4ae9f497eedf75d9fa3d072e9d4351ea8eec845f14dbe877b","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8c08c2576548098891e1714e","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2dc8c3b86c2301e8e36747dd192480e4cc0ceb1c7ed0287f37152c179354d01d","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":22913},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Gemini 3.6 Flash (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7c277219a5bc4d8aa9be404603e80d2af8287c89f47308be017697cd144d8faf","result":{"confidence_high":-0.0662523433492713,"confidence_low":-0.08741234081641375,"sample_count":1523777,"score":-0.07683234208284252,"score_display":"-0.0768","source_stated_rank":45},"run_fingerprint":"eabb921c38aecbf77a788f2386eb0cae671fddd25efaa2d060230039afc8a5a4","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"50449acbaf909ec2f0bf0422ee811f645d7fed38344528eb36b3f71cd03cf2e1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Gemini 3.6 Flash (High) · gdpval"},"model":{"id":"google/gemini-3-6-flash","name":"Gemini 3.6 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-6-flash","upstream_model_label":"Gemini 3.6 Flash (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"9e231f72e450874eb64c34fca3cf1a5efe49f2ba2ebfd9eecb105a896fbd2ff5","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1264.67,"score_display":"1265","source_stated_rank":null},"run_fingerprint":"bac11b15225a04f8a18f1c5c394205fd37022fc9aee563a159e5bcd706e00c94","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.6 Flash (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b910cf792225dccb483afa4f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a7153c275fdaed4c79a509c772ca56d9f0ed64e9d1b3f305639355b75ee0d389","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-09"},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Muse Spark 1.1; model release: 2026-07-09","source_accessibility":"API access","source_model_name":"Muse Spark 1.1","source_model_release_date":"2026-07-09","source_model_versions":["muse-spark-1.1","muse-spark-1.1_high"],"source_reasoning_efforts":["high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8b8bfc00d604e7ee286b61d350ac8f2b3849a8df5917d7e810313de5d6ea473b","result":{"confidence_high":157.1,"confidence_low":152.17,"sample_count":null,"score":154.31,"score_display":"154.31","source_stated_rank":35},"run_fingerprint":"66cd4321d9439a0461d6c2ebbd6e23e2760a53dd47e472e8c806a3b95bc3ded2","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cd729d89537d94a3078f527f","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fadeb542e86671274da5e2e6400d12ed3d5987167cfaf0ee206ad88feb689970","metric_details":{"license":"Proprietary","variance":5.065742208762903},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1495.7506877788974,"confidence_low":1486.9280260283335,"sample_count":37238,"score":1491.3393569036155,"score_display":"1491","source_stated_rank":12},"run_fingerprint":"6120aaa936f16c89c120e0c82c6c815ea7f4e1c8b9b2d1481dccb4c1b0d19b19","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_99814c21bc9361f199e3a520","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1a0aa6235709e8259c65b0a65d670f7c8869ca88ef210e095d7411509bc5f745","metric_details":{"category_scores":{"Agentic Coding":58.535333333333334,"Coding":77.1585,"Data Analysis":72.548,"IF":69.6375,"Language":74.342,"Mathematics":87.14474999999999,"Reasoning":87.73075}},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.29954761904763,"score_display":"75.30","source_stated_rank":37},"run_fingerprint":"61eacebe97a9fc72c7cf182840d5f8bec9c6ae4552367dae3347972e09d5ede4","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-09","basis":"evaluation_at","evaluated_at":"2026-07-09","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"34272963f1a255e23d3a7525f678a28edd208a2a37fe40e21377f77381fb2c0d","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/meta-muse-spark-1-1","source_model":"Muse Spark 1.1"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4af2544e468228c0ae8c94b37bb85399d9527d09f75350cc2badda8609a813dc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.2,"score_display":"52.2","source_stated_rank":3},"run_fingerprint":"4ce906c27a5c7e17f069b59dcda1c7b1cac75562cecb33ba752bd7ecc7fd19f5","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1cb2c5290629fc013609d211","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-08-31T19:21:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-31T19:21:58.000Z","first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2c323303e79b013f55745d3cd2e625798fa3271b57db2824d033dbbee18a93cc","metric_details":{"best_score_across_scorers":"0.5778894472361809","model_release_date":"2026-07-09","stderr":1.5665429072238852},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bCdwpRXgZtvQLYC4wA9JXr","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbCdwpRXgZtvQLYC4wA9JXr.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bCdwpRXgZtvQLYC4wA9JXr.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"c8e8fa50c510fadf6b63e74fddf3368f09db7944e2accd9801f2362c09998e52","result":{"confidence_high":60.859368821776904,"confidence_low":54.71852062545927,"sample_count":1000,"score":57.78894472361809,"score_display":"57.8","source_stated_rank":17},"run_fingerprint":"ec70d226ce3d3330d6765cb35595f65bf8843303b5068ce5bed1fdbe56f7b4b5","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a5dc1bcf8ba60512cb3ee6a5","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8e8596af2829088da6677ddfe80b80409ebefd47be913cd8a59b3ce07f5c1a47","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.034304,"date_is_proxy":true,"latency_seconds":75.198,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.104},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":90.404,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":256000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"xhigh","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark_1_1","zero_shot_accuracy":91.919},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"2f5a7c4745e5d483468d7300372b8bbe97abb9b9c920d703ac61d06bd7666130","result":{"confidence_high":95.28584000000001,"confidence_low":87.03816,"sample_count":396,"score":91.162,"score_display":"91.16","source_stated_rank":23},"run_fingerprint":"f3d112f50b69a34115944ef37737dedd722cd32303f865be1a09707c53c16728","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ae1f4b9253a5bd5767eaf79abfc511a79f22ea8b3f6567dabdf9baa8ce640ea0","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.010704,"date_is_proxy":true,"latency_seconds":21.03,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.318},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":256000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark_1_1"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9f72c78f4ea2553d297cb1fd14b9c86e317ae99e573f73dcbfc7c31c7c523328","result":{"confidence_high":89.34928,"confidence_low":88.10272,"sample_count":null,"score":88.726,"score_display":"88.7","source_stated_rank":17},"run_fingerprint":"7d34a14b7ff4486e2b55a9f2a32dcd217a8de2818187d1808bdda1ea436c38af","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bb49f0ee572ac37296edeb26131084b295ffeee27d45c406805f9bab2dfd8e6f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.01361,"date_is_proxy":true,"latency_seconds":31.715,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.819},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":256000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark_1_1"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"dd3eb4ec3da9e66fd4250f6a6f475efad25434b09da4f2f9c96910ac61c9aa3b","result":{"confidence_high":88.19524,"confidence_low":84.98476000000001,"sample_count":null,"score":86.59,"score_display":"86.6","source_stated_rank":18},"run_fingerprint":"9693b75bf221b95a8938b924856d2f62bcf4813df83350cf0cedcb9f6e3746af","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"38e3d0d41a815b20909b291032cae404969794e8b04588b6494f5c1fae91096c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.1","source_effort":null,"source_model_version":"muse-spark-1.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b8d5681d3a43da044668078b0086d17dd08f304be440c7a42c4a4ab7a3bb0205","result":{"confidence_high":25.198836,"confidence_low":8.58398,"sample_count":71,"score":15.1,"score_display":"15.1","source_stated_rank":78},"run_fingerprint":"8969b4664e1443f4b5225ad8770ae3af5327793c3538008d1a0f58cddda51c3e","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_623a805a6834d32cfd32240f","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5adf541fdce9481e2ea5452fdc53b70a1192cb44739eb8923c203ffe70eb6424","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.352022,"date_is_proxy":true,"latency_seconds":209.115,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.72},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":256000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark_1_1"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"58dbbfb884fd9df39bcdabd00071aa929fa2ad5691988392b3c856071f4c182c","result":{"confidence_high":85.3712,"confidence_low":78.6288,"sample_count":500,"score":82.0,"score_display":"82.0","source_stated_rank":21},"run_fingerprint":"4faa693ea6d3be408285246695b49a5c9143230bf1f8bdb5491673296e27b290","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"fdcc066f6b139ca82842e9053f1ff4383019e648a3c2a588c39aac61d680c8b7","metric_details":{"ci_attempted":452,"ci_half_points":3.0353424539337115,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":241,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":95.82522123893806,"mean_cache_tokens":11781947.471238937,"mean_cost_usd":2.361142644469026,"mean_duration_seconds":897.5133296747788,"mean_input_tokens":12005399.261061948,"mean_output_tokens":74008.4203539823,"median_agent_steps":86.0,"median_cost_usd":1.7171158,"median_duration_seconds":688.5861825,"median_input_tokens":8037064.5,"median_output_tokens":66088.0,"median_output_tokens_to_pass":61398.0,"median_peak_context_tokens":154817.5,"n_attempted":452,"n_passed":241,"n_tasks_attempted":113,"n_tasks_passed_any":90,"pass_at_4_points":79.64601769911505,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_muse_spark_1_1_xhigh","source_model_version":"muse-spark-1-1","source_reasoning_effort":"xhigh","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"9fa9b172911f42872d2217a905d99d07616d897ce35fabb291c5e9ab2403da64","result":{"confidence_high":56.35392652473017,"confidence_low":50.28324161686275,"sample_count":452,"score":53.31858407079646,"score_display":"53.3","source_stated_rank":46},"run_fingerprint":"d1c241e03c555e41c0fa68926303824db73297f307756a1d8a0576db8f0dd4ee","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"658f98689012cff3f0f02f5ec5e6befa8cf0bd9fece3ebe44960e2a5df60775a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"95.82522123893806","mean_cost_usd":"2.361142644469026","mean_output_tokens":"74008.4203539823","model_release_date":"2026-07-09","notes":null,"pass_4":"0.7964601769911505","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"muse-spark-1-1 (xhigh)","source_effort":"xhigh","source_model_version":"muse-spark-1.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"20cfb75c80afba3aca51fc7f9cc81157f3e8f1432faefa704b4f1a054adf311f","result":{"confidence_high":56.35392652473017,"confidence_low":50.28324161686275,"sample_count":113,"score":53.31858407079646,"score_display":"53.3","source_stated_rank":45},"run_fingerprint":"b460efae302eef01ba772040062d6e4e35b9acb3647d53307b9290f16a300d83","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":87,"at":"2026-07-09T18:04:14Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-09T18:04:14Z","status":"fresh"},"measurement_id":"3b71c494b540ea79d2abb95a1f0e369813fc6258f5a48c8ab7a18e1d4d727463","metric_details":{"confidence_interval_half_width":3.1,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=Muse Spark 1.1*; createdAt=2026-07-09T18:04:14Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2026-07-09T18:04:14Z"},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":true,"cost_cap":"uncapped","protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"Muse Spark 1.1*","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576","turn_limit":250},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"5a1cf05b09868affb81eee80ce0dd47bc3d0235115de220e12653dc752c0054b","result":{"confidence_high":64.6,"confidence_low":58.4,"sample_count":731,"score":61.5,"score_display":"61.5","source_stated_rank":1},"run_fingerprint":"c078af91521b3e2851dca77e64123da617b0ee1c11fcdafab06d800a021d4da2","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=Muse Spark 1.1*; createdAt=2026-07-09T18:04:14Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7d87524808b5d9b5715e4a3e","provider_config":{"source_id":"scale-swe-bench-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"040b6167e8aa49428f08ddaadbac791d4380601dddf06e8abfc7e7162c157ebf","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.046497,"date_is_proxy":true,"latency_seconds":113.253,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.992},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":256000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark_1_1"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"6a10ec42151a2ccc6c2a2dcda56bbe4264b86945967c3206b0c0a48e2fcf4566","result":{"confidence_high":87.85032000000001,"confidence_low":83.96168,"sample_count":null,"score":85.906,"score_display":"85.9","source_stated_rank":29},"run_fingerprint":"93d02c4ea2f60cc4adf956c50cce1c70c50ff8b16b746f66fa34ee568d561f2a","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a5f85b1abbb1203456fdb3f8444f2126a74e4ff1f034845aea7e75912945e096","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.1 (xhigh)","source_effort":null,"source_model_version":"muse-spark-1.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"f4114cd93d65e5ba85c390cb39604c1c1e33071ef0d1a146068d6ffb208d1060","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.796296296296305,"score_display":"58.8","source_stated_rank":15},"run_fingerprint":"e93680c14f8cbde5707c28b3ffb30d73fa78c6977c700d1c3efa96f037c1f327","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4997d2bedea1e95dc121dbfc18400b595427f068260cdda3f933378164d5ccf9","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.923283,"date_is_proxy":true,"latency_seconds":615.428,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.327},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":256000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark_1_1"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"e117aa8cbb00e928b904e9bdbdf6bdc3c5d5dfc72b66b624cb3c17b17ced560a","result":{"confidence_high":78.67892,"confidence_low":65.63708,"sample_count":null,"score":72.158,"score_display":"72.2","source_stated_rank":32},"run_fingerprint":"e0e2890bc4572d1b827b781e7c8df15c307837cf1b3d1cf79b8d2d2fe99d4275","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"eb63a5855319a86b2895439685bddac98caef83db5d7a188c837318b58c6190a","metric_details":{"license":"Proprietary","variance":14.992952680718636},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"muse-spark-1.1","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6230503c36b90ff1ada487c57594ded4e9c94a2c85b42cfb0847171e5ab08e88","result":{"confidence_high":1549.2442304279123,"confidence_low":1534.0659814744135,"sample_count":8070,"score":1541.6551059511628,"score_display":"1542","source_stated_rank":39},"run_fingerprint":"cddb8c4ce8d8bfdc7fa8bc5be626bd8bb28d771ebe805d627f0d714cf3136f19","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f91f386d6cb697210b7a2893","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6e941abc994458e2306d9d1ddd5cee2b0a185e25fd506f5c35f556077c47e177","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"muse-spark-1.1","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"e1d715260f2814a51603722611dbd7de42af5f304a174ef2c6ef9303e8062057","result":{"confidence_high":1549.34,"confidence_low":1533.75,"sample_count":7549,"score":1541.55,"score_display":"1541.55","source_stated_rank":36},"run_fingerprint":"f50e64accc9885180ee33c4c604f9ea754284bc2365997ce6921289ac7e32ca4","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_40df47b5931fb824b86ab803","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"216355fd5ed17fad9b006f0eae442241a87f88f6890c03cbe0d6681812b818fe","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.417396,"date_is_proxy":true,"latency_seconds":624.772,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.991},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":256000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark_1_1"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"2f6772a2ec3e813e242b3fa81f712261f586eee1fe1a9fc08dc5dfedd2aabc6c","result":{"confidence_high":71.23035999999999,"confidence_low":67.34564,"sample_count":null,"score":69.288,"score_display":"69.3","source_stated_rank":30},"run_fingerprint":"c2dcb319d03a16209fb73e726f67fea0ba6c0f23825003c8ea005598668031c9","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":74,"at":"2026-07-23","basis":"evaluation_at","evaluated_at":"2026-07-23","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0e91c66ff82ce8d867d7f2b5ba2bf507a9e931c9c42443d6a46788d3f708c6a4","metric_details":{"cost":null,"pass_2":29.55326460481099,"pass_3":23.969072164948454,"pass_4":20.618556701030926},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"muse-spark-1-1_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"586a45dfded2cfa3639c1530e9bbc6ff3221b066b8305b9464d7b46e95b10f2c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.4639175257732,"score_display":"40.5","source_stated_rank":null},"run_fingerprint":"1f2f66d4bdf86391211f8260d98f2d6c9ae3484dd3fb44bbf46679733af6abeb","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":87,"at":"2026-07-09T19:46:54Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-09T19:46:54Z","status":"fresh"},"measurement_id":"05ae4388a669e4382a5213cf49f00dd94f5171b73c6b4c2a51a6d5d40bee15af","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":1.95,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=Muse Spark 1.1; createdAt=2026-07-09T19:46:54Z","source_row_created_at":"2026-07-09T19:46:54Z","task_pass_coverage_threshold_percent":75},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"Muse Spark 1.1","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"bad1307f4750a0643e6c2fe9689c4e9fc29015704e460567735f771f266918f9","result":{"confidence_high":90.05,"confidence_low":86.14999999999999,"sample_count":1000,"score":88.1,"score_display":"88.1","source_stated_rank":1},"run_fingerprint":"77a502058f2b8ecf8d02b31e16d738bb7a039c3694df1a99fd0acf1c6ef9d972","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=Muse Spark 1.1; createdAt=2026-07-09T19:46:54Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f46411f67d3ab003209ce38a","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"b7572fb44b0a1508280cd34df19760d667368f3ca6bbb47e1d216cad7dfb6b24","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.5810000000000001","mean_standard_error":"3.4","model_release_date":"2026-07-09","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.9,"upstream_source_url":null},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.1","source_effort":null,"source_model_version":"muse-spark-1.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"1be2235594083bc4e8191cc91abb8a7c5bfe7afbf1b6258b92e2ede04f106e4e","result":{"confidence_high":49.544,"confidence_low":34.256,"sample_count":480,"score":41.9,"score_display":"41.9","source_stated_rank":7},"run_fingerprint":"e7c69ead38c7932a7793957cd14d637ee0800988418282618ea60ab3ea59e78d","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_458ad66cb249e06fdb6aaf95","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"162fb1c8c25c4cb86ec4eda12ef46b8a13b73323da6c324660b9fb823d468ce7","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":142711},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Muse Spark 1.1","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bfd49d80597a6aa6a53e2cf1e1cf8fe072e64a0f621c6390d9baa7ddf873220e","result":{"confidence_high":-0.04437910672810896,"confidence_low":-0.053302077309958236,"sample_count":7612456,"score":-0.048840592019033596,"score_display":"-0.0488","source_stated_rank":39},"run_fingerprint":"8f9338adc70f7c3e45afa038a6af92a3a9d4e452de1778f5aadea387e96fb9e8","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_60c2da8ae56f193d786d0708","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2c748c60a07caaccd4eaf47293b120bb3ab775dd26a6eb210b013f96580014fd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-09","notes":null,"upstream_source_url":null},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark 1.1","source_effort":null,"source_model_version":"muse-spark-1.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"b622d2048bbaf7ec329beaf3505277f8648ca0c8997f8289c3e57a872ac18f59","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":6520.474999999999,"score_display":"6520.47","source_stated_rank":14},"run_fingerprint":"cddb59764377cf2fb1cee3d78dac3d8071d0d5e2945cd109e1d083e0fc51a259","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0cc4cc6b4e71e0063d4a49e2","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2e1a4d2762cde679245a44122da24154ada48704d7d9c26430e71db8783b3378","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1048576,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-09","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Muse Spark 1.1 (Xhigh) · gdpval"},"model":{"id":"meta/muse-spark-1-1","name":"Muse Spark 1.1","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"muse-spark-1-1","upstream_model_label":"Muse Spark 1.1 (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c73e7a675c35d96ece2a9994877c8c6034e424b37eb1bd387045cf42049c1192","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1208.36,"score_display":"1208","source_stated_rank":null},"run_fingerprint":"815b81d27c79ddd1aeca0a042371fcbd7ba5af9ef606a45b9caab45e242d147e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark 1.1 (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_f65a7f664c0eb4433d5d5397","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b3d9e4e1fb119e6f3c074bf88e48494faa3abd08643375b5eced0d3e2288886a","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-08"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Grok 4.5; model release: 2026-07-08","source_accessibility":"API access","source_model_name":"Grok 4.5","source_model_release_date":"2026-07-08","source_model_versions":["grok-4.5_high","grok-4.5_medium","grok-4.5_unknown"],"source_reasoning_efforts":["medium","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"95878368e2e4ad34ab67394e3635ac11ae94edbecbcfd5805702ea2bb4fd4453","result":{"confidence_high":156.08,"confidence_low":152.34,"sample_count":null,"score":153.99,"score_display":"153.99","source_stated_rank":36},"run_fingerprint":"169a0e140545c1e39c44b56094d80f7cef19e02e52f9fc4be3aad0d767d2ba61","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cb44654bc85aaf39fe593479","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5c5d37b73c584b3f5c5931ae8ca6cc5bb9bf63d14e848b034a55b8fa10eab68e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-08","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.5 (high) · intelligenceIndex"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-5","upstream_model_label":"Grok 4.5 (high)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"94b68681b50a24e9f9dfb303169e59c3f2848b25c7edbe4433a5fd0e69972cb1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.08424,"score_display":"39.1","source_stated_rank":null},"run_fingerprint":"5660718545ff5a94da54996f21bb8ac26372e0e0c92c82ddfa81a7ff4b3f4a19","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.5 (high) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0cfdb864912d4993572b20aa0a8b4957801ffc1916ac92d8c2e41ef91f9213b4","metric_details":{"license":"Proprietary","variance":4.917454478268999},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1470.1322231898132,"confidence_low":1461.439651906031,"sample_count":39789,"score":1465.785937547922,"score_display":"1466","source_stated_rank":50},"run_fingerprint":"57e9d81142ebcdfc6169b69950d2b30e2a50da6bb1bbaebd668986293ccf9b1c","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7c1e5a04c3bf319f34e54098","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"657ee8f012688869bf5d421800d0795065d2d516733662b20b69bf314a1b0bdb","metric_details":{"category_scores":{"Agentic Coding":56.464666666666666,"Coding":68.5855,"Data Analysis":73.03766666666667,"IF":71.52925,"Language":82.79533333333333,"Mathematics":90.8245,"Reasoning":87.173}},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.77284523809524,"score_display":"75.77","source_stated_rank":35},"run_fingerprint":"3ffed0f9f5afbca1a813e51008ea9afb70bb7853f4057c0d57fa498aa5a5bb04","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4a583e289c2d0896454de86f","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"acc7ba410e25c51db2d428aeb6ea191e94f0b98176a1b5b8f2bc168899624213","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-08","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.5 (high) · hle"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-5","upstream_model_label":"Grok 4.5 (high)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9ed249e49ef2d928fe3f70ad57eb328a3ff34b2b25fe8281440621d9c32f4087","result":{"confidence_high":44.776463,"confidence_low":40.60637,"sample_count":2158,"score":42.678406,"score_display":"42.7","source_stated_rank":null},"run_fingerprint":"34b88495ef8a7de06426180d152c4330363080c5b1888153f57d362e06ddb686","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.5 (high) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:29:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:29:30.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d48e50db4a667714452f985b6fab450cdc3fab175885a982eeda7427a729a3c2","metric_details":{"best_score_across_scorers":"0.483","model_release_date":"2026-07-08","stderr":1.5810153729833274},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7HWW3sjRstKzb9JpcnvLaV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7HWW3sjRstKzb9JpcnvLaV.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/7HWW3sjRstKzb9JpcnvLaV.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"708704be34be87713cdc078eb35d334a987d1955cb120dd9aa7ecf74278c21a9","result":{"confidence_high":51.39879013104732,"confidence_low":45.201209868952674,"sample_count":1000,"score":48.3,"score_display":"48.3","source_stated_rank":30},"run_fingerprint":"e107129db619be3759f108fff55a79dd659a6f2edd4fa476622c2c0535dd018c","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-22T23:07:14.159346Z","basis":"evaluation_at","evaluated_at":"2026-07-22T23:07:14.159346Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"62cf4474b7adae1e6026e264a021cc299638c557f30f6e218425addbe917db68","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"grok-4.5-0708","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"51fb334a5199590969bdd2fd7c4cd270ba514ee382bf5d0c7ec67c5aab1f62ca","result":{"confidence_high":56.98238824544852,"confidence_low":50.646477733932926,"sample_count":null,"score":53.81443298969072,"score_display":"53.8","source_stated_rank":9},"run_fingerprint":"8db1841643e2a6f415ab6327141768ebd4dcac0131992360c090da4a0c8705ce","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a27f36541d8d75a03e46652b","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4952399c365b0ece8f8706f370c4e2f3dd121138a9610cdb67103abe85788439","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.044029,"date_is_proxy":true,"latency_seconds":217.858,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.288},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":92.929,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":null,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.5","zero_shot_accuracy":92.929},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"b8fc7dc1668d5e6944209576b7cf8b2eb6445440135a50f67bcf7934db0959e6","result":{"confidence_high":95.45348,"confidence_low":90.40452,"sample_count":396,"score":92.929,"score_display":"92.93","source_stated_rank":12},"run_fingerprint":"6629f3aab76ab4a9419bd7ce13bc1cbaed2730c3b4e96740e22f66401fd6ec51","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"db4383f3f73188a80934a34637ffeeb7d2c0e5ba5e42c9399c0f9c46457ffaf6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-08","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.5 (high) · gpqa"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-5","upstream_model_label":"Grok 4.5 (high)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0ad71a4529be8b42196ed7de1295f917df65f31303cff3ca1f24af8b059eaf91","result":{"confidence_high":95.894948,"confidence_low":88.725863,"sample_count":198,"score":93.131313,"score_display":"93.1","source_stated_rank":null},"run_fingerprint":"3ca5b8e2a4ddd9d648dc5de602bd9f459b7cbb53574418913a8859c44d18802b","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.5 (high) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":88,"at":"2026-07-08T23:54:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-08T23:54:22.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"408456a9b750b8e1924054e5f9d07c33b88bc0805ac9efd63673e907de3cd361","metric_details":{"best_score_across_scorers":"0.9343434343434344","model_release_date":"2026-07-08","stderr":1.4340395485738813},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"9kSgmUd9vh4CBXz3tJhctN","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"fe073d7515f7ac4e736d6d4e0c0a9d061f08298662cdedbd0928c000fed49e13","result":{"confidence_high":96.24506094954825,"confidence_low":90.62362591913862,"sample_count":null,"score":93.43434343434343,"score_display":"93.4","source_stated_rank":16},"run_fingerprint":"c7f7ed05baa5d58abba7516ca90134275acf0634c88dcda532f70ae3c9cdf6ac","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":82,"at":"2026-07-14T20:29:33.683293Z","basis":"evaluation_at","evaluated_at":"2026-07-14T20:29:33.683293Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8eafcb9f8be5c4f0fb0b17d0e1a8a9defc4afc40982f05e3292f4cf3ffb5313e","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"grok-4.5-0708","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4f0cc37ca8d4ebc403ceba696af67df176e665f7662964b555015572d795732c","result":{"confidence_high":97.26290708398315,"confidence_low":90.61588079480472,"sample_count":null,"score":93.93939393939394,"score_display":"93.9","source_stated_rank":1},"run_fingerprint":"e4f3b16d3aa54c31833a72853dc36df121684fccad0ddce06020d982ac613bd0","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c9c9e4a6c497352249aed04d","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bb122dbc87f98a06dc3a58094979f7c26a216b373d2bf62331672b85a2feccf7","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.014974,"date_is_proxy":true,"latency_seconds":357.751,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.31},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":null,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"467585e482dc3f86e4055738b27acd804afa6f917d548cc57fb08582a2279fcb","result":{"confidence_high":89.8236,"confidence_low":88.60839999999999,"sample_count":null,"score":89.216,"score_display":"89.2","source_stated_rank":14},"run_fingerprint":"6706b4307fc2e4680212686042bd782165e3c001118ab53f37c3fe1db72e74ea","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-08T23:54:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-08T23:54:22.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a0b5504508b75033be3ea8a6c1868c93246f6b06a4037e996bf0a04a59d988f0","metric_details":{"best_score_across_scorers":"0.9777777777777777","model_release_date":"2026-07-08","stderr":1.2743145865697103},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"f8Pac6hqib4t7x2s4HCRoT","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7cbf38020bd69bd48d8e8fe29843f017d6e1e5a9a361c88d9b00f8b0f310957f","result":{"confidence_high":100.0,"confidence_low":95.28012118810113,"sample_count":45,"score":97.77777777777777,"score_display":"97.8","source_stated_rank":25},"run_fingerprint":"1a784ae68cf1b111944a812e745d95339542831f7f63478c97a455385deb168e","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T01:53:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T01:53:42.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7f3e280df6049105f031950c91e3c80083b3762bd0fcad295a02471ada68ce69","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5719298245614035","leaderboard_private_problem_count":285,"model_release_date":"2026-07-08","public_example_count":10,"stderr":2.9360921879029935},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"djbbEqvBW9qux3pZwcZN2i","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Grok 4.5 (high)","thinking":null,"upstream_model_id":"grok-4.5_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"b6614fa95b24c1d92a5abd67890f0e700c4381fe4735c9c1acc943007f927de0","result":{"confidence_high":62.94772314443021,"confidence_low":51.438241767850485,"sample_count":285,"score":57.19298245614035,"score_display":"57.2","source_stated_rank":39},"run_fingerprint":"9d87b8e641a7bc84c6a9023452a1a3a71bbed637bc08c9777cd719e38f92e293","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f7986e3ecdf1d28c44ab16870064dc89c00b7f4709ebd0266ec7e3cf6accbe40","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.78,"model_release_date":"2026-07-16T00:00:00.000Z","results_url":"https://arcprize.org/results/xai-grok-4-5"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"xai-grok-4-5","model_type":"CoT","upstream_model_id":"xai-grok-4-5-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"71362c525a715c2754a7cff2c2545649c1c0aff25800e6118476191022c6b207","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.64,"score_display":"53","source_stated_rank":93},"run_fingerprint":"dac626fcd9ee85091a6ef75b098224c625fa13ec9c07d7feeb54682bcfb00ebc","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"beccaeb3dec976062a8aa18c26a07192f0bc379edd8815dce6f70d222283851f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.75,"model_release_date":"2026-07-16T00:00:00.000Z","results_url":"https://arcprize.org/results/xai-grok-4-5"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"xai-grok-4-5","model_type":"CoT","upstream_model_id":"xai-grok-4-5-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3ba054d99c431d1ab0a32058a61d3d576f892f968717aba7fb5385226eb56ca1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.64,"score_display":"53","source_stated_rank":93},"run_fingerprint":"efc3e1921ddb06a559254be3edc11c1658e363cf4fcf682c0fe29fac33b21371","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ed465d7ae13ec2a579ab6273","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"050d5b6b78176e2612054c3ba8904f3eafa81442b8c8815d9b2edfd3941b869c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.4,"model_release_date":"2026-07-16T00:00:00.000Z","results_url":"https://arcprize.org/results/xai-grok-4-5"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"xai-grok-4-5","model_type":"CoT","upstream_model_id":"xai-grok-4-5-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6086b4417d16decf5ba1d6547b35c470119fb231f6c567545eeb316e286a212a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.06,"score_display":"33","source_stated_rank":113},"run_fingerprint":"bab2fad0b4473f797e6270799da8420fe8ac89b7e739698868ab372e062682a2","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_888d68131ecc8fd7cb2ecf1f","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c7c655ee23b57685b64895d8dbadb673a282dc4b873c341778374159f26e2257","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.78","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5 (High)","source_effort":null,"source_model_version":"grok-4.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c9e456d964f8cb08cabfe5ed03a7da8d8bbba59a0048a80d6a0a31c3379f9cf6","result":{"confidence_high":57.742749,"confidence_low":47.481503,"sample_count":360,"score":52.64,"score_display":"52.6","source_stated_rank":96},"run_fingerprint":"71e552e79d9ce5ae91cffab024b861efd8d84c970435e84ae30293d327e0aa0c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dd8be6751ab12953b43cba06f8e260423320085e62dde16e1bda02a7ea2f42e2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.75","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5 (Medium)","source_effort":null,"source_model_version":"grok-4.5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"03962289b57d0ab13157d5e7d06c288aee848b8fad06621c9c4842842d218012","result":{"confidence_high":57.742749,"confidence_low":47.481503,"sample_count":360,"score":52.64,"score_display":"52.6","source_stated_rank":96},"run_fingerprint":"4df3548c3aeeb4c903acf451ad286b651fa00dbdcabfd176dbcb6a8465d498a5","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ed465d7ae13ec2a579ab6273","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3adaac739912ba5d7180ef82ef900c82c530800c9302f8220394c6ff4b018e15","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.4","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5 (Low)","source_effort":null,"source_model_version":"grok-4.5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"1fa92abafd113c5af0325a1f1f4cb387e5e1b8feacaa6d60615474acbfb7ff1e","result":{"confidence_high":38.076029,"confidence_low":28.401691,"sample_count":360,"score":33.06,"score_display":"33.1","source_stated_rank":113},"run_fingerprint":"f90c0b2eb0dd650920f2d2385a40a99a1b9e56db3400122e3fc6e652301fc94c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_888d68131ecc8fd7cb2ecf1f","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-09T01:53:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-09T01:53:42.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aa23fd116cc6bfdb54b932d3a2fcbd8d974edfc50d54c56ebae3c84a7ddf7072","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.24390243902439024","leaderboard_private_problem_count":41,"model_release_date":"2026-07-08","public_example_count":2,"stderr":6.789956540036613},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"7vThbrtbczScg2UcsAYXtF","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Grok 4.5 (high)","thinking":null,"upstream_model_id":"grok-4.5_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"0cd213659d79ae2397b7ff1939e91844219063f5993ae240a46e4183d90e0845","result":{"confidence_high":37.69855872091078,"confidence_low":11.081929083967264,"sample_count":41,"score":24.390243902439025,"score_display":"24.4","source_stated_rank":42},"run_fingerprint":"06e886106fc4743a042ccfa764d54a504afdd5f0dde59137a76cf46b9c5746b4","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b034ab07aa89e87c097114efb1e0b77cebb06b43de30757c5b52eaf084713393","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.030117,"date_is_proxy":true,"latency_seconds":714.648,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.168},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":null,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ebb04f33e7fcbb571fea2800d80c0d3209a9648e1acfa7825412f5d1691cc8cd","result":{"confidence_high":64.08128,"confidence_low":59.502720000000004,"sample_count":null,"score":61.792,"score_display":"61.8","source_stated_rank":83},"run_fingerprint":"54be0897a8505adf56c83521d0941bf232528787408420d4ff4897e44fe4cc33","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e4978945bd4afc4bb10851cc5d555e1a529d4ae67cdd14a15f8c8e2dc822f31f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-08","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.5 (high) · mmmuPro"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-5","upstream_model_label":"Grok 4.5 (high)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"3134f40fd5da727d680de2c6ebdf3855c8bbd87ce7607f7a7d8be55959a5381d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.404624,"score_display":"80.4","source_stated_rank":null},"run_fingerprint":"8e93e96b4e2dd29e49d723945b3f6ef028d30ec49f126fdd6adf93e2b4c576b2","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.5 (high) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7a0aae11ab7d4dd353d7b397d1ba89c20a9685dacfd67b13ef5b57f03663cd92","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5 (high)","source_effort":null,"source_model_version":"grok-4.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"bbc176aa802f7b25971c5051ea30b46347cd09f13ac5423fc7f5f0560a46760f","result":{"confidence_high":25.577174,"confidence_low":8.829054,"sample_count":71,"score":15.4285714285714,"score_display":"15.4","source_stated_rank":75},"run_fingerprint":"ab3e19bc8ff59711898b63a631976982288a5b161506d60ac655d02ebec74cd5","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aadaf6c6f1221ecae2c6383c3e3eac16893cd3299f84db6b7ac8721119e8bbf0","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-08","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.5 (high) · critpt"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-5","upstream_model_label":"Grok 4.5 (high)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"ea039b0f7fbdb90ec2692e08c47019b4b0e53416048f6eff211e3ee551495f75","result":{"confidence_high":25.660281,"confidence_low":8.79401,"sample_count":70,"score":15.428571,"score_display":"15.4","source_stated_rank":null},"run_fingerprint":"55e8ab2d8a97dda003c72ef02312e945f0d309ece1e61b8e0a2c4e1e51cc1271","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.5 (high) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c4d787afc68a5ccdf4f426a28fdaee5fe096406754d6e08f77c92b44bec1f4cb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4.5_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"7eb48decea3ecaa512c14af260f93c5830a0a514400310caae0df1b4f8cb7406","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.0,"score_display":"70.0","source_stated_rank":14},"run_fingerprint":"529265da377c238a54a7fc32ca29829675ca25a3bf9eb968b35bbceb9d24c95e","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e2a20830119d2bc9ecee925c","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":88,"at":"2026-07-08T23:54:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-08T23:54:22.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dbfd0609a4cbc8d61ef963517aca942f721adefba7926148df948dff97bc2c24","metric_details":{"best_score_across_scorers":"0.36","model_release_date":"2026-07-08","stderr":4.824181513244217},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6e3LDVuZwoZC4Sg7ushoVP","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"920a20e591b869b75fc8f8d4158f94aba644ef8b7b9332d61428d05cf9ea9d55","result":{"confidence_high":45.45539576595866,"confidence_low":26.544604234041337,"sample_count":100,"score":36.0,"score_display":"36.0","source_stated_rank":39},"run_fingerprint":"6d6c1690b4fed93e65d7027621e5568965dbda0a038aee440ea7e3303c6ef512","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e1d379bb400d79c4282afd605b13db9b4ff95edb5af14c782ecfc41e9d1f405f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.33","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5 (High)","source_effort":null,"source_model_version":"grok-4.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0be15b5fd304eb56ca0f44856d57eb910ed57bfeb05acceaa66240d6bd25a138","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.67,"score_display":"85.7","source_stated_rank":102},"run_fingerprint":"5ef0b42cd7194a13978c8421afa3c43cb957638aed8acad1e3a5f944911df491","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"da312efdf510254e91900bbbb057daedc5982d227eabe0d33a4cdf037d64b121","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.31","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5 (Medium)","source_effort":null,"source_model_version":"grok-4.5_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0fae3df8a1a03b7ea32ef8c82ffe97954776a79a554dd605fc317a7b30860362","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.17,"score_display":"87.2","source_stated_rank":87},"run_fingerprint":"4e078c05e413a4d7b510906013083051640bb273ad67fee32b3dec34799bbbd3","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_ed465d7ae13ec2a579ab6273","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"76291fa746630d0cdd481396e0236477f3795b9a45c690177132ab8a54f8699a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.18","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5 (Low)","source_effort":null,"source_model_version":"grok-4.5_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ba6dd4674df33ca6ba8ed5ee8a4afd08090dc1395f52a9784dd69ef5f6f4369e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.17,"score_display":"79.2","source_stated_rank":114},"run_fingerprint":"1259e3a5ad4d4b835ae11d124f280486327392cf7e03c47064eba93b37f72384","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_888d68131ecc8fd7cb2ecf1f","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"255a439b2bee15d18f7c5ac1447e0cc05431a330dec1a71a61bc84f8f92a1f7b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.54034,"date_is_proxy":true,"latency_seconds":199.573,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.525},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":null,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.5"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"616172393bf1d7bb76f99a2cc608e77ec6116f8124ed15ce9891068b10128da3","result":{"confidence_high":89.589,"confidence_low":83.61099999999999,"sample_count":500,"score":86.6,"score_display":"86.6","source_stated_rank":13},"run_fingerprint":"1f16baefbaa86e5fb9a546a4e6c891f233b8464258e3f7486554984bf191d4db","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4ebe742cebb311519ed2faeeb4b7875656723738dd21e53b562f5e6ed8946c89","metric_details":{"model_release_date":"2026-07-08","pass_at_5":77.47747747747748,"potential_contamination":true,"sem":0.5975900523162895},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"contamination-risk","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"Grok 4.5 [high]__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"97b9c1a9295d5e896a33151e74769b7d9c82eddee23bbc29b0cdc9a0768fb11c","result":{"confidence_high":64.95506028632371,"confidence_low":62.612507281243865,"sample_count":null,"score":63.78378378378379,"score_display":"63.78","source_stated_rank":2},"run_fingerprint":"b62926607b49aa632167d53a21be5bbc9bdfeb32369ec136fbf5ce3dbc4f5dff","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"contamination-risk","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"03fdb0b16f77c8d43b91f1c1f844c1e2645c3f0d14f22efd0084630ddfa61b76","metric_details":{"ci_attempted":452,"ci_half_points":2.28080457287792,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":243,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":61.33185840707964,"mean_cache_tokens":3943846.5132743362,"mean_cost_usd":2.4157481725663716,"mean_duration_seconds":484.7029359800885,"mean_input_tokens":4059182.975663717,"mean_output_tokens":35525.33185840708,"median_agent_steps":56.0,"median_cost_usd":2.02311,"median_duration_seconds":430.0852805,"median_input_tokens":3293744.5,"median_output_tokens":34250.0,"median_output_tokens_to_pass":33872.0,"median_peak_context_tokens":94044.0,"n_attempted":452,"n_passed":243,"n_tasks_attempted":113,"n_tasks_passed_any":88,"pass_at_4_points":77.87610619469027,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_grok_4_5_high","source_model_version":"grok-4-5","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"92509b6905548ce6b2e9db8427bb6bde696ae1d9d7d8b138aa5e5c11b885c7f7","result":{"confidence_high":56.04186651978057,"confidence_low":51.48025737402474,"sample_count":452,"score":53.76106194690266,"score_display":"53.8","source_stated_rank":43},"run_fingerprint":"98c6076e0851e228353e6ec4bbb40052ddca9b08593f86502d28236cef4cfc63","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e010d7bd7b49b87b3590e08c2263c314fd66b088eb12412461f9519e140fae20","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"61.33185840707964","mean_cost_usd":"2.4157481725663716","mean_output_tokens":"35525.33185840708","model_release_date":"2026-07-08","notes":null,"pass_4":"0.7787610619469026","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"grok-4-5 (high)","source_effort":"high","source_model_version":"grok-4.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"3b08cd85469992c8795cf1fb8b883a6555980dd4e3f535b3c865ac576c5cd1a5","result":{"confidence_high":56.04186651978058,"confidence_low":51.48025737402474,"sample_count":113,"score":53.76106194690266,"score_display":"53.8","source_stated_rank":42},"run_fingerprint":"f53dcea8279027778a29721baed31c057ccc42e64805e56f8c900ea1fe421175","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_d16705309178e80bc97af71d","split_or_window":"vendor-public-own-harness","unit":"percent","version":"SWE-bench Pro · SpaceXAI provider run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":81,"at":"2026-07-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:18Z","last_confirmed_at":"2026-10-05T12:33:36Z","observed_at":"2026-10-05T12:33:36Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-16","status":"fresh"},"measurement_id":"9eecac1594a9f3953cde9bdb3e06ed38bf17448cb5eb6c2abb8ac062cf15a37f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://x.ai/news/grok-4-5","evidence_sha256":"5d79e853f58b703e537bac20aa0f4491672d93cb63447e08cc0b182cdc55f20a","kind":"reviewed-static-report","published_at":"2026-07-16","reviewed_at":"2026-09-08","source_url":"https://x.ai/news/grok-4-5","version_id":"0405f681bf173b1a0be35498dece50deea2555163e889dd0dc115184b8959a5e"},"source_locator":"Real-world engineering benchmark chart · SWE Bench Pro · Grok 4.5 bar and accessible chart caption"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"SpaceXAI","evidence_verified":true,"freshness_proxy":true,"protocol":"SpaceXAI provider evaluation; exact task revision, run count, scaffold and reasoning level not disclosed","reasoning_effort":"reported","result_published_at":"2026-07-16","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"9718cff3d299c8e2a23caea567e672ed829f7c8a4dff327f416fd99989395f18","source_published_at":"2026-07-16","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"SpaceXAI coding harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"b0dcc8aa5dd29a0d6e7d22d852ed5919db981141055a099570b3643c308d103e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.7,"score_display":"64.7","source_stated_rank":null},"run_fingerprint":"57b8e4b4897fe621d8d16fd32629dbad6f9fd83ca939b0bb85e647540ffe420c","source":{"content_hash":"7d5b8786fee10354efb239c7319ad3c8a8602363a23e95db42d800cc49d877b8","fetched_at":"2026-10-05T12:33:36Z","first_fetched_at":"2026-10-02T19:14:27Z","homepage_url":"https://x.ai/news/grok-4-5","id":"xai-grok-4-5-report","last_fetched_at":"2026-10-05T12:33:36Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Real-world engineering benchmark chart · SWE Bench Pro · Grok 4.5 bar and accessible chart caption","name":"SpaceXAI · Grok 4.5 launch report","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://x.ai/news/grok-4-5"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c7d36fce95c61967be262faf","provider_config":{"source_id":"xai-grok-4-5-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4009d0cf3d3b22aeacfb66092e3328c3bb1133aa60138a4fd9618df41a92a8e1","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5","source_effort":"high","source_model_version":"grok-4.5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"grok-build","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"40f16d3459455495eb47f3033dd5b0fd57b39ce300086fbc9e2ced61045e5710","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":42.44,"score_display":"42.4","source_stated_rank":17},"run_fingerprint":"e05b17cadad9bcd8c6c13ce56f4590e82022db0eb18da43348644811fd8485e3","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1fea3ba1eee7d83152167d1ce5599d30a59c9ad0f9d780ee21a35a9ad6e19b7e","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":5.637814,"date_is_proxy":true,"latency_seconds":5194.818,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.951},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":null,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"high","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"ef4f1903b00a6851d8d0a091277266d42632c741ca9595781a51db7be2564730","result":{"confidence_high":48.29996,"confidence_low":32.812039999999996,"sample_count":null,"score":40.556,"score_display":"40.6","source_stated_rank":30},"run_fingerprint":"868947f6b4f69b3acd78ed9424abb6d8ecc993fbfda14795f4742ffbfc193834","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7aa155b9039fec436126e2a42ce80117632505e54e951cc607f54c1aad2345bb","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.045235,"date_is_proxy":true,"latency_seconds":589.356,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.961},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":null,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"ae5cbcced42ed010d640eff9f630f1717719370695776cab76cb7d6746c81c78","result":{"confidence_high":89.23456,"confidence_low":85.46744,"sample_count":null,"score":87.351,"score_display":"87.4","source_stated_rank":15},"run_fingerprint":"8a046c07a913d79d876ba560fffa3019a75321153169248e04b65f58768acf08","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"635993eb1bf16941cd684512df58960e1b22e930c095bfdf4c94609ea1f80834","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5 (high)","source_effort":null,"source_model_version":"grok-4.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"530bdf4a1f6314167fcb831176fd410f4fb3271748853ef539a893f44eb57243","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.050925925925895,"score_display":"54.1","source_stated_rank":55},"run_fingerprint":"4f00d302626ae185ffc5e7782068822985fe73490e7c55b860e15911721b1140","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"afe2bc466e22efa876bc990b62968e36af4eecaf34d478fec012f987b225ff43","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-08","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.5 (high) · scicode"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-5","upstream_model_label":"Grok 4.5 (high)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"731bce59b276eaf3a57a8810ca329ea5790ad74193fc21906029bfcb81152f3a","result":{"confidence_high":60.6198,"confidence_low":49.202881,"sample_count":288,"score":54.976852,"score_display":"55.0","source_stated_rank":null},"run_fingerprint":"d049189766fc80d62642a3f71532c7f7c11a3a8a3e05b5a5d6930f6aa97e750a","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.5 (high) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"86c3d893ae0b06fe8aaed15c418aa20db841c3fbd35fe437f4375fb8af78547b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.176767","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"grok-4.5-high","source_effort":null,"source_model_version":"grok-4.5_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"5e43d648a673b4fb850af65f22c490376fc0f5a52012a0a59dd674757e7382f8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1309.28,"score_display":"1309.28","source_stated_rank":27},"run_fingerprint":"92b535b6721501d95d71da1ca4f688b57e80301d112f50a2ffeaf650a8d860eb","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"44cc1d1c02129cdc746e2d8705f5b42b49ca95109ad28a84342dd2f7c75092ef","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":4.102194,"date_is_proxy":true,"latency_seconds":830.057,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.48},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":null,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.5"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"a71a64890f8c3fadce99c7fb6bfcdbed468fb179840ce9e1978c79d0109eb41b","result":{"confidence_high":77.7828,"confidence_low":60.221199999999996,"sample_count":null,"score":69.002,"score_display":"69.0","source_stated_rank":37},"run_fingerprint":"e45b9d2e2c37797a0cf516bc3dec51b1e78abd62b8146bfe62145f1a8f09f2af","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bea7256f413e9c5534fd515e245b8ef14cc3c833d3aebe654c9cc6683a47ff48","metric_details":{"license":"Proprietary","variance":11.435275750890368},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"grok-4.5","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fa871030515cdae0c66dea0a3f6a9915fae9398ed8f9700e6df0c006c2d0a93f","result":{"confidence_high":1558.4489979566283,"confidence_low":1545.1933363887808,"sample_count":11393,"score":1551.8211671727047,"score_display":"1552","source_stated_rank":37},"run_fingerprint":"49c3ae4efe44eb51f4a19570d88b283658609f51b6d157a50c16fa54b8f5cd80","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0a2cd2202522f2c34abea7da","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"68cbc5c498d5f0a9f271e99bd0bce4154928d43d45a83410bacb9845ab8daae8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4.5_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"1f38b435cc79407c05609fba1f83f3f9942e597847d9314616253c68ef9b256f","result":{"confidence_high":1562.67,"confidence_low":1547.54,"sample_count":8133,"score":1555.1,"score_display":"1555.10","source_stated_rank":33},"run_fingerprint":"7d266b6dd4eabf97c41fbe4df38a4aa4cb138d1dff96062bc27abc2ef2efa38e","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5d3f732c10f8e01fb46448a0","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5f83d0527e9bc0323abf2337a7fbfae896fec02b97b630a20bcf463aea707f70","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4.5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"2b35c9e6af9f4c3d2d0c9da17a6e48ecc93aa147e7c71b994af82185394750e5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.43,"score_display":"46.4","source_stated_rank":83},"run_fingerprint":"dc30ba2cd9b21c8437bdd82cdb8c92ea91a6dacb435f6d3bda87d65d8690d679","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bc00c246c21133ce08995d8c","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"564d38a49c1dfe01699dbba2fc4861d87397928ff25f23a37f098098b7407821","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.283943,"date_is_proxy":true,"latency_seconds":624.415,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.416},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":null,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.5"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"4c9c4e289dc982c9525545142cc15ac3de4a75ecd9d9d48d61eaa4a728cd742d","result":{"confidence_high":76.44536000000001,"confidence_low":59.134640000000005,"sample_count":null,"score":67.79,"score_display":"67.8","source_stated_rank":32},"run_fingerprint":"299608463d39a73ce0c61fa9877f7f3012c00ad6582c33ae73296c16b14bca23","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0c820b706f49dea397e027ba37702be68072630edf3b1e1c45335433188b9bec","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-08","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.5 (high) · terminalbenchV21"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-5","upstream_model_label":"Grok 4.5 (high)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"f2bdae3d2441a29becb64b714c45883469654ed35a68f56138481b49661518d2","result":{"confidence_high":88.320641,"confidence_low":72.356182,"sample_count":89,"score":81.64794,"score_display":"81.6","source_stated_rank":null},"run_fingerprint":"dbbbc0041a539be9d09a2347dcdee6e68eadf22e810de2d8856f95cc390a19f1","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.5 (high) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T21:39:34.661007+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T21:39:34.661007+00:00","status":"fresh"},"measurement_id":"5c089362da083b33cf9a25499c6a3d560c2f76e2002cd285e9f26504335c9dee","metric_details":{"accuracy_stderr":1.47,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:07:26.20393+00:00","total_cost_usd":766.02,"total_tokens":1238993118},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"Cursor","url":"https://cursor.com"},"agent_url":"https://cursor.com/docs/cli/overview","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"e660765d-7b96-48f7-97ed-3c2abe17dbb5","model_release_date":"2026-07-08","model_url":"https://docs.x.ai/developers/models/grok-4.5","n_trials":370,"source_row_date":"2026-07-15"},"run_count":5,"scaffold":"Cursor CLI","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"caad2167d6277230a98c08d572a03818e1f18c1cbb389f66f957eb8652e6e5ac","result":{"confidence_high":25.627121,"confidence_low":9.120366,"sample_count":74,"score":15.68,"score_display":"15.68","source_stated_rank":9},"run_fingerprint":"327d9c632e04a2ae64e7b61a9765629ef0d2e0b70fed6d0254fb3c60f995a547","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_cf7ce74acf6af29a63ba09f9","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":73,"at":"2026-07-24","basis":"evaluation_at","evaluated_at":"2026-07-24","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7d4802eb3595b848cffe7bfa78d55a5c88a88d7ca0b6a10ef09fc44294abc9d2","metric_details":{"cost":null,"pass_2":39.51890034364261,"pass_3":35.30927835051546,"pass_4":31.958762886597935},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"grok-4-5_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"5e904dbd73f67cadb131197389e5eea603a8f90d31e89d354571d853b5d36e5f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.93814432989691,"score_display":"47.9","source_stated_rank":null},"run_fingerprint":"10fc56b054be29add97ca2e5dc29371bd4e0a1d4817ad5f00f709f6d2fc9019c","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"625b2a24498e0b19ab259b2c30fe06756fea4a30fc40432673f4f44a17ef5554","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-08","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.5 (high) · tauBanking"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-5","upstream_model_label":"Grok 4.5 (high)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"75d8bf7afe74687378deb70dd0c118ff0b09997f4dbbda5c6f173943d12a2ea1","result":{"confidence_high":52.004251,"confidence_low":32.724274,"sample_count":97,"score":42.061856,"score_display":"42.1","source_stated_rank":null},"run_fingerprint":"dd328c124b4f720cddeb19ca8aee35e0d4fd0a4feb1d2df1d789187d7eae2182","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.5 (high) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"10c89349eeb34a92e27caf2d3316e35389aca4113a4e9b32aad1e7eb6a085f77","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.705","mean_standard_error":"4.3","model_release_date":"2026-07-08","notes":null,"standard_error_points":540.0,"upstream_source_url":null},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5","source_effort":null,"source_model_version":"grok-4.5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"c3968653b34fcf31f82a0265a454d0bc70c257d43997f5c5ea616332eb1bbd24","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":56.2,"score_display":"56.2","source_stated_rank":15},"run_fingerprint":"2d4c6f6ce16e382868e201a1d7756afabfb33dc87eba26c5f3069c846445216b","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7b5a51c768e676f4fd84a86a","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"50deac423e3b801ee6af1488d9e79d4be93ac181e99fb306d286812abc149b2f","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":41920},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Grok 4.5","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e38c198bb6b4ee4888e71702a661d171821906b757911a8e39a123123ab9e410","result":{"confidence_high":0.02220602180284431,"confidence_low":0.0019617931260805486,"sample_count":2418629,"score":0.01208390746446243,"score_display":"0.0121","source_stated_rank":26},"run_fingerprint":"e88daf2053d82bea1e1fb0810783dce88559ad896e30fb74757b86576a564264","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e977802683ce940c35f0c1f3","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"88824567932dfea61cc81f78dcda691f5d5a69768c75507f647ce9f4fba678b5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-08","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.5","source_effort":null,"source_model_version":"grok-4.5_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"a9af951f1de03bf9cc633f76e5734942b582f6a9d28975187544eeb7ee252bc5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3887.4259999999986,"score_display":"3887.43","source_stated_rank":38},"run_fingerprint":"b9d7b79bfe418e47762f1ba23b1c7eb165723e8c60c78dcf68190183bf6040cb","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_22b572a08df9ceebd792a8c8","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f98d7cb8d73e50c0e118ab2a9900383b03c50a841f7127ce1100d056cd1e2052","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-08","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Grok 4.5 (High) · gdpval"},"model":{"id":"xai/grok-4-5","name":"Grok 4.5","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-5","upstream_model_label":"Grok 4.5 (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"19beb440861d4544038f22e9cf28c16a73d97df21132ddd799c781cc479b2881","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1370.44,"score_display":"1370","source_stated_rank":null},"run_fingerprint":"fda18fd18233d7f8bd110ad7a04fb96cb393da9bc1a5e7dc24ed8aa05b5d33d3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.5 (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b7ffbaf8d89bbaac429d0b87","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0d6a31603fc04e2388ec8a788cfb0b889c69bf0bb9a9781e38be68671ae90d4a","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-05-19"},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen3.7-Max; model release: 2026-05-19","source_accessibility":"API access","source_model_name":"Qwen3.7-Max","source_model_release_date":"2026-05-19","source_model_versions":["qwen3.7-max","qwen3.7-max_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bac6dd9c00cf7f60e134c0fe08b3ca0da2f5f41eee86d324c8269f31c2da3e0a","result":{"confidence_high":156.02,"confidence_low":151.88,"sample_count":null,"score":153.72,"score_display":"153.72","source_stated_rank":37},"run_fingerprint":"b467dbdaed82089cc9e7bec40da6050f9873014b7fc25ddad49e38f0850925a2","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_438a0adfb228d31d4956122c","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e283e510cd34278f8a114576d9a2d37cf36b7dea6e9d7f46c9f6bdd00541a377","metric_details":{"license":"Proprietary","variance":24.7237137872618},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1484.62782899041,"confidence_low":1465.1367922392883,"sample_count":3920,"score":1474.882310614849,"score_display":"1475","source_stated_rank":35},"run_fingerprint":"aa08b9683d58f18bcbb85ac34173b8711db4cedcc87387cf66aea782e5c8daec","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4220a4a6f1311bc74f0c6484","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"6fc8ac25eeef542bd519663a8e1a073ae9834ac2f84695c4d384202dac976763","metric_details":{"category_scores":{"Agentic Coding":43.586000000000006,"Coding":74.219,"Data Analysis":71.786,"IF":74.0415,"Language":79.73899999999999,"Mathematics":85.24825,"Reasoning":83.3365}},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.13660714285713,"score_display":"73.14","source_stated_rank":47},"run_fingerprint":"a21d670b579aba2cb322dce176fcc81844640452a0ea4b4393d480004d994069","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8a98f0234f2d8eb2f775b7dc","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:34:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:34:09.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a8e62e5e11a9aa7f86323f656c1910d46ffd28f5851c1280ad24aa94b7c72148","metric_details":{"best_score_across_scorers":"0.5576730190571715","model_release_date":"2026-05-19","stderr":1.5737359397180177},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"DMjUrG6eZEqwcQ4SqUoViu","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDMjUrG6eZEqwcQ4SqUoViu.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/DMjUrG6eZEqwcQ4SqUoViu.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"eb1583e703edeb823f406ba50e4126f2c85021e5a69a322f58b085a038b90fba","result":{"confidence_high":58.85182434756446,"confidence_low":52.682779463869835,"sample_count":1000,"score":55.76730190571715,"score_display":"55.8","source_stated_rank":19},"run_fingerprint":"e1a603d094c83384e6526cae155939ab1ea4ebe68d960ba76f64957cc1cc20fe","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3a75cadcf4f028e0f62f9064","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"78496a9f1ac229deab610172a31c9403ea87340f053a4d47c8030a17922d10ef","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.06149,"date_is_proxy":true,"latency_seconds":126.611,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.497},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":89.899,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.7-max","zero_shot_accuracy":90.404},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"83da1d731d305de4a2d633aac747aea72c57c7878c49e17b2cc40ea0bf907a2a","result":{"confidence_high":93.08612,"confidence_low":87.21788000000001,"sample_count":396,"score":90.152,"score_display":"90.15","source_stated_rank":25},"run_fingerprint":"4b2c153c665ccbedf27a48a154fc1fd1c4b6eb5e867d62d4026959fb7a86f556","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c851e77e82debc794fff6ed8","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:46:28.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:46:28.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3ec3db2988d53b6fac5dfdc823d13d0d14e4f9c6132e3c5147747833943a080b","metric_details":{"best_score_across_scorers":"0.9090909090909091","model_release_date":"2026-05-19","stderr":2.0482086775424224},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"W6mTanyjvXhasFBzHukv4S","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FW6mTanyjvXhasFBzHukv4S.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/W6mTanyjvXhasFBzHukv4S.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"c541cbf846c3b0217ac5b4eda1ff521270801cc540e24218f7f73e23b3e798fa","result":{"confidence_high":94.92357991707405,"confidence_low":86.89460190110776,"sample_count":null,"score":90.9090909090909,"score_display":"90.9","source_stated_rank":34},"run_fingerprint":"fe1c576001cc3e495a6b7d154db6818267be156ac3b52f2ab378c5e2e5ca941f","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_68a81c056b9a4eea7b217404","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"33a389b7b7db625f4013e80c752bf4991edfbced00f8b445322abc9f0560c6be","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.01678,"date_is_proxy":true,"latency_seconds":29.15,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.305},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65536,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.7-max"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"74c844bfb22b365f973ad30162f735165086d406ebec3279c387020df3c84750","result":{"confidence_high":89.90880000000001,"confidence_low":88.7132,"sample_count":null,"score":89.311,"score_display":"89.3","source_stated_rank":12},"run_fingerprint":"6873a22e933fe7e207df671681d1efea3e50ba1c5196205aa4faeeabd3a9c041","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0e54016d3db95a8f54d1a2e5","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:39:02.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:39:02.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ad9fb3cdc4fd25646746d4feec3c8c8d42828990391dcd106f32e4436f4682c3","metric_details":{"best_score_across_scorers":"0.9555555555555556","model_release_date":"2026-05-19","stderr":3.1067790907534736},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"D3YDUZSfEUDsS4frLbEKDu","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FD3YDUZSfEUDsS4frLbEKDu.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/D3YDUZSfEUDsS4frLbEKDu.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a5b341a2ae4d57ce93a4f520cbabc07ca59484bef7b4675d88563352687d0e12","result":{"confidence_high":100.0,"confidence_low":89.46626853767874,"sample_count":45,"score":95.55555555555556,"score_display":"95.6","source_stated_rank":38},"run_fingerprint":"ef9e3fcdfd2ebf66fefc556bbef177c758e8bdc5fa73bc249cb3f67341eb8d69","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b3e2c828e764c38eff97fcb8","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-13T03:08:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-13T03:08:22.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"35735ee65ae40889aa60567f3973822a5fdf9c014798c2044f17ee8bc4dc446a","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.6456140350877193","leaderboard_private_problem_count":285,"model_release_date":"2026-05-19","public_example_count":10,"stderr":2.838347520543562},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"RG2biwnmGybSVTVxxMdRWa","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRG2biwnmGybSVTVxxMdRWa.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/RG2biwnmGybSVTVxxMdRWa.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.7 Max","thinking":null,"upstream_model_id":"qwen3.7-max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"664220cdf0cfc70a17f9aa1a388d66675817155056d69c7725a6617df98310bb","result":{"confidence_high":70.12456464903731,"confidence_low":58.99824236850655,"sample_count":285,"score":64.56140350877193,"score_display":"64.6","source_stated_rank":32},"run_fingerprint":"5e5782ad6b31a933bdbcc69f7ed0466d07aa9acce41c8754e42d993d5432051b","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4a082360cdbfb26695a8efcc","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-13T03:08:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-13T03:08:22.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"711e819ff51ee09ce338b5c3437bc0744c78a6b1d8de32a28c71496adf914c32","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.34146341463414637","leaderboard_private_problem_count":41,"model_release_date":"2026-05-19","public_example_count":2,"stderr":7.497768853141171},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"FHZCKbtfvpKGXU99vZFnXN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFHZCKbtfvpKGXU99vZFnXN.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/FHZCKbtfvpKGXU99vZFnXN.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.7 Max","thinking":null,"upstream_model_id":"qwen3.7-max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"792de66b59f8316ef0e0a3ccce21e0a763caa123578ca4cff217300a967b5b1a","result":{"confidence_high":48.84196841557133,"confidence_low":19.450714511257942,"sample_count":41,"score":34.146341463414636,"score_display":"34.1","source_stated_rank":29},"run_fingerprint":"c76c124296645dc2c6f09738b188c9a38850afe6a1595e3032ddb6ba3b59b7f0","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_898133c7fa458202db96273c","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b7fec2cdc5e4a9b92da6d7f4fe0135caa75b340ab25e327d146fe49d1dae5761","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.7 Max","source_effort":null,"source_model_version":"qwen3.7-max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"91f50faa87fd0d841f330a8c7b2b3a0bce11c7f101afbcf432c347910ddd411a","result":{"confidence_high":23.255382,"confidence_low":7.356164,"sample_count":71,"score":13.4285714285714,"score_display":"13.4","source_stated_rank":82},"run_fingerprint":"ca3702effa1193e9b67beee3e173d4a868115b47a1d22a36fc19c94e380fb5ae","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d772c2aef798e310edb6ef13","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a5c6c967bc46be1272d2a7d436c5d48b9037401a629c74850257d6ec3765a77b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.7-max","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"29beccbaf009967b3fb5b2f9d46ebc0d6bb778cceb0260b854dd809068370284","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.4,"score_display":"70.4","source_stated_rank":13},"run_fingerprint":"cfdc0433bd4680a1e5813d46182c520bfb59ca34df6c5d9eee55399c8828bce7","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_73d6450b88bda658f569c477","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:46:56.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:46:56.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"38eccaca08d8a2d07a7e9987da26007bf130a66193a850539c21bb683344d5c6","metric_details":{"best_score_across_scorers":"0.19","model_release_date":"2026-05-19","stderr":3.9427724440366254},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"JfkPdo29um2NkxdhT79AK2","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJfkPdo29um2NkxdhT79AK2.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/JfkPdo29um2NkxdhT79AK2.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"eacae7fa31b7c3717992aa4fd836ec09e946eace315139763994af950724fb56","result":{"confidence_high":26.727833990311787,"confidence_low":11.272166009688213,"sample_count":100,"score":19.0,"score_display":"19.0","source_stated_rank":99},"run_fingerprint":"2a73e83f69dca68b892dadbac11c26c818e935cc49141c03c82e677c2ca793c0","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a74bf66befbcb7f81dd6ab02","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5086626eff5076c26ac95b5f2f6d63b4e0c745f38d82a9a83573ff1e6b90dbfa","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":3.227207,"date_is_proxy":true,"latency_seconds":976.131,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.074},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.7-max"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"ac36971307414b334b1236cf5b4cfd25c85add8be2df29f6cb6b144668b25a50","result":{"confidence_high":72.86504,"confidence_low":64.73496,"sample_count":500,"score":68.8,"score_display":"68.8","source_stated_rank":69},"run_fingerprint":"ff1f9c6bbaf6bef156a28b6e2ee0b5ccbcde1ebcd52ea14df6c3fa4738715f98","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8702741fcb32199baffce119","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":108,"at":"2026-06-18T17:48:14.587Z","basis":"evaluation_started_at","evaluated_at":"2026-06-18T17:48:14.587Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"db3d7c1af97f36cefa9fc0011c5bb6a6aa954db77e9b363ab2639d932dc22357","metric_details":{"best_score_across_scorers":"0.7727272727272727","model_release_date":"2026-05-19","stderr":1.906835447169652},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"83hhDhcT9UUrZ2NiUbWSUU","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F83hhDhcT9UUrZ2NiUbWSUU.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/83hhDhcT9UUrZ2NiUbWSUU.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"b40d3302de8d973bf951ac3792bf4a1173a90977105184dccecd8baab01f1647","result":{"confidence_high":81.01012474917978,"confidence_low":73.53532979627475,"sample_count":484,"score":77.27272727272727,"score_display":"77.3","source_stated_rank":7},"run_fingerprint":"a2455b5775396598e51ca1333b8ed2c68c37c22b2aaef17d7fda137875c569f9","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9cdd961f4572bb5b3d5364ba","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_0f94a5c34a87744f357d4816","split_or_window":"qwen-vendor-comparison","unit":"percent","version":"DeepSWE 1.1 · Qwen comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":54,"at":"2026-08-12","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-12","status":"fresh"},"measurement_id":"1729584b15d82d058282a9ef97c57cf522f9d2c7b5d81e4685df92b5e5c0dc2d","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Coding Agent table; DeepSWE 1.1; Qwen3.7-Max column; methodology note 4"},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on DeepSWE 1.1; highest score across Claude Code and mini-SWE-agent; temperature 1.0; top_p 0.95; 256K context; reasoning effort not otherwise disclosed","reasoning_effort":"reported","result_published_at":"2026-08-12","source_content_hash_method":"raw_bytes","source_content_sha256":"7690c680c2d6eb284179ef5dd4db63260d248531cae9087b09b2c2200230f331","source_published_at":"2026-08-12","tools":"Claude Code and mini-SWE-agent tool environments"},"run_count":null,"scaffold":"best of Claude Code and mini-SWE-agent","tools_allowed":"Claude Code and mini-SWE-agent tool environments"},"protocol_fingerprint":"26f7dbbbd59c10993e81bb31ec607ed81e91dce338e64d5c3ca6702cda935db1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":21.6,"score_display":"21.6","source_stated_rank":null},"run_fingerprint":"0e2a61089a7bdf299d96d35ea04a1681b730f592a7667daf9ce0042a44fe6079","source":{"content_hash":"60e41c5cda83e653de6d55a51d96311dc9f5cca01af54cbccb44db4fcdeb4295","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","id":"qwen3-8-a95b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen3.8-Max model card; metadata facts only","locator":"Coding Agent table; DeepSWE 1.1; Qwen3.7-Max column; methodology note 4","name":"Qwen3.8-2.4T-A95B official revision-pinned model card","redistribution_policy":"metadata-only-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0d904d7fe29e536b30a70704","provider_config":{"source_id":"qwen3-8-a95b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"metadata-only-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_232b90e5b13ff602749b9048","split_or_window":"qwen-refined-vendor-comparison","unit":"percent","version":"SWE-bench Pro · Qwen refined comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":54,"at":"2026-08-12","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-12","status":"fresh"},"measurement_id":"cfb79d525525b48ac469af7cabab0c7029f48b20178f2dbbcc5dcb6617a65898","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Coding Agent table; SWE-bench Pro; Qwen3.7-Max column; methodology note 3"},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on the refined SWE-bench Pro task set; Claude Code; temperature 1.0; top_p 0.95; 256K context; task count and reasoning effort not otherwise disclosed","reasoning_effort":"reported","result_published_at":"2026-08-12","source_content_hash_method":"raw_bytes","source_content_sha256":"7690c680c2d6eb284179ef5dd4db63260d248531cae9087b09b2c2200230f331","source_published_at":"2026-08-12","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"ca23444044809f8156938debc846e7b2fff27eb273a37bfb51447632d47704b2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.6,"score_display":"60.6","source_stated_rank":null},"run_fingerprint":"7601648d7a1443b0858ac74db8a9563e567718cfd05d5af52cc2405d9cf0b499","source":{"content_hash":"60e41c5cda83e653de6d55a51d96311dc9f5cca01af54cbccb44db4fcdeb4295","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","id":"qwen3-8-a95b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen3.8-Max model card; metadata facts only","locator":"Coding Agent table; SWE-bench Pro; Qwen3.7-Max column; methodology note 3","name":"Qwen3.8-2.4T-A95B official revision-pinned model card","redistribution_policy":"metadata-only-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0d904d7fe29e536b30a70704","provider_config":{"source_id":"qwen3-8-a95b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"metadata-only-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a5df046135603c4ef2977964ca345274030469dde65c2ea0e6d717258a9f8f90","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.086751,"date_is_proxy":true,"latency_seconds":192.175,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.974},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.7-max"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"01f2ea212cedacb2bcaf08470026e212ea333d2e01f07d3b8ef66217dafb6a72","result":{"confidence_high":88.96604,"confidence_low":85.14796,"sample_count":null,"score":87.057,"score_display":"87.1","source_stated_rank":19},"run_fingerprint":"53bf99754eb12059e2f65c107651d50bbb7b4d1d86912badac574256dc24a232","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e9712f59624c456abd520aa2","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ea98e3aed8dbcfa7d248e1842679755bc5c5962fb56ad74576aecc7dadb320d0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.7 Max","source_effort":null,"source_model_version":"qwen3.7-max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"c9f86708fcaa36df1d06c5b20b1487ea19b6b4ed0166ce1d63b6cd30bb0ca964","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.842592592592595,"score_display":"48.8","source_stated_rank":100},"run_fingerprint":"39cabea9cf45bfd920904dfed3095265c1fb4d9eb0b603b5afb6106a06d66b85","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d772c2aef798e310edb6ef13","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"70a66dee074cc7216df62a1dc6fc43e365050969d125618462eb0990aa3de667","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.3845","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"qwen3.7-max","source_effort":null,"source_model_version":"qwen3.7-max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"921e8716214d093a8a220fc7e1c823ee9f3af73c6aebebe12122685d9c21764a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1189.4,"score_display":"1189.40","source_stated_rank":36},"run_fingerprint":"c180b107d8d542ad3d79772f0b7b94d355c1b1ec5e6083656ba43b77cf610e20","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d772c2aef798e310edb6ef13","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aa1538265d9c562810776f21a9f3c5d743270067da67d706f2268e678235429a","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":11.400322,"date_is_proxy":true,"latency_seconds":4689.627,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.63},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.7-max"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"9ee33ad7c0a2e25a1c813f226007278b97344f3b93ade90acea3c36ba84dccba","result":{"confidence_high":56.7458,"confidence_low":38.596199999999996,"sample_count":null,"score":47.671,"score_display":"47.7","source_stated_rank":56},"run_fingerprint":"ea669a9f69740d728c8b0e58992255e5eaaf221b2e80c2635571e492e099db8f","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5db0d7d1f8becd7348837d1d","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"96f5be1f78f800e940a666ed0f055215808500cefe5e6c2ba9a451b80e255300","metric_details":{"license":"Proprietary","variance":14.086901978074344},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.7-max-20260517","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8f0b49b47bce8e7a836b8fae652b627b9b8ff453c7a296485af705a663301c0b","result":{"confidence_high":1522.2893354874157,"confidence_low":1507.5768572651816,"sample_count":8922,"score":1514.9330963762986,"score_display":"1515","source_stated_rank":49},"run_fingerprint":"353dc2a5759c122c1079920f095c20994c9b204fdfbbf60d1b0aa2e4b43cabad","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_93f8ad037c4bd04e2de7d0e0","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d0baadf5b8cd43a2ba4f2194dae592eb23c0bd0d5b59a6d880205a31b9548865","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-19","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.7-max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"4756a01af5aa63a16f97e60c00261b8b3d46fb410e4dce4425dddfcecb8c9fcf","result":{"confidence_high":1524.7,"confidence_low":1509.54,"sample_count":8363,"score":1517.12,"score_display":"1517.12","source_stated_rank":46},"run_fingerprint":"95de0d727d69be7cce97af1da07cfe0577295bcf4dd15f2c75d070a287f85d34","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9e47a7f808b03dd19049f3d9","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"03af8d0967b0a21c3576e46b2bd5d5c5fe56bb0a5b12d121ebcc3cff10295c41","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.33233,"date_is_proxy":true,"latency_seconds":935.196,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.375},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.7-max"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"bfd59e8b57cd74e6fc1d70a212c68cbbfc17c7e4f5183583b44f0505d3fd56f8","result":{"confidence_high":61.784,"confidence_low":60.314,"sample_count":null,"score":61.049,"score_display":"61.0","source_stated_rank":39},"run_fingerprint":"42567c2a35dc06893a63ae97c0ed25e13a42ac2da2e569258d1ce2a983acb44c","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9834bcc9b3e87e87a66577e8","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_ff4a6e288c3c17d588478b46","split_or_window":"qwen-vendor-comparison","unit":"percent","version":"SkillsBench · Qwen comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":54,"at":"2026-08-12","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-12","status":"fresh"},"measurement_id":"0b4458942c5517b7ff7da7a12ca1e61d96bd31558641f769e07e419205119317","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"General Agent table; SkillsBench; Qwen3.7-Max column"},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card comparison on SkillsBench; task subset, tool policy, run count and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-12","source_content_hash_method":"raw_bytes","source_content_sha256":"7690c680c2d6eb284179ef5dd4db63260d248531cae9087b09b2c2200230f331","source_published_at":"2026-08-12","tools":"not reported"},"run_count":null,"scaffold":"not reported","tools_allowed":"not reported"},"protocol_fingerprint":"5a87a249627548ba41ab1e22adf8f465931b323f1bc4cc1e11b5f7988d5b1a3c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.2,"score_display":"61.2","source_stated_rank":null},"run_fingerprint":"e365ccca63745a886ccffc98dd77d5d3a48460c04323f088738897867f70d2f9","source":{"content_hash":"60e41c5cda83e653de6d55a51d96311dc9f5cca01af54cbccb44db4fcdeb4295","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","id":"qwen3-8-a95b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen3.8-Max model card; metadata facts only","locator":"General Agent table; SkillsBench; Qwen3.7-Max column","name":"Qwen3.8-2.4T-A95B official revision-pinned model card","redistribution_policy":"metadata-only-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0d904d7fe29e536b30a70704","provider_config":{"source_id":"qwen3-8-a95b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"metadata-only-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9de6c9e9b8cb657949f33a428625591a95808637212ffdf759eeae6db6681e91","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":40438},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Qwen3.7 Max","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"aea68a78ad4f298d2ff3abdeac5246559105ddc0fd5a2610df9fb0382dac8ae3","result":{"confidence_high":-0.041974256927270914,"confidence_low":-0.060609075119189545,"sample_count":2207521,"score":-0.05129166602323023,"score_display":"-0.0513","source_stated_rank":40},"run_fingerprint":"c915fdb8454e3ac9780bca2f471625c8223a1b2f0532e6213c03401a5be4a860","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a11f0381d740fab28f4a990c","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1b4b9de8e1bfb6b9641cf0cb263b73869aef79ba5d09e61f6d7c5a7ec3e345c6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-05-19","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Qwen3.7 Max · gdpval"},"model":{"id":"alibaba/qwen3-7-max","name":"Qwen 3.7 Max","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-7-max","upstream_model_label":"Qwen3.7 Max"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"312f662adad3422e6c306c3d9c5daa7cc3ad434b6530812aa1feb179b9cdeb8d","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1114.28,"score_display":"1114","source_stated_rank":null},"run_fingerprint":"6fc131fc0185105293d9174f65c18dd7256621804b34b12a22e87e634fdd4121","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Max · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b653e6b09c5d08a78703143b","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f3fc5fd36117bdfe89b9940872e8113e1d8bb4631c24064b510d27741ace2e13","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-12-11"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.2; model release: 2025-12-11","source_accessibility":"API access","source_model_name":"GPT-5.2","source_model_release_date":"2025-12-11","source_model_versions":["gpt-5.2-2025-12-11_high","gpt-5.2-2025-12-11_low","gpt-5.2-2025-12-11_medium","gpt-5.2-2025-12-11_none","gpt-5.2-2025-12-11_unknown","gpt-5.2-2025-12-11_xhigh"],"source_reasoning_efforts":["off","low","medium","high","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9bb864fac210d71859bb6b0be60ad52c34ce0614cbf132e78c2786b5b7cf4b86","result":{"confidence_high":155.37,"confidence_low":151.7,"sample_count":null,"score":153.49,"score_display":"153.49","source_stated_rank":38},"run_fingerprint":"d90fc7b2cb31a7969d6cfbcb1a4190d0cef704b6a66ebbe7f8ee5528e11eaf81","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d195c64b8a3ecc15e17b76c2","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1132216cc49ed621ac7a3f00b59c04efe2d8e88e02663247c10593e8690bc162","metric_details":{"license":"Proprietary","variance":3.2997388997412997},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1440.830330700315,"confidence_low":1433.7097096312232,"sample_count":49542,"score":1437.270020165769,"score_display":"1437","source_stated_rank":101},"run_fingerprint":"f05db52e827541317b7b137295d7bdc217527d27c7d8912cd4ec232bf711d992","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6f7d981cfef38dbe68ebc84d64cb77a62410eff41e491b933c5249c87092dd1a","metric_details":{"license":"Proprietary","variance":2.630462299009703},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1438.771414366636,"confidence_low":1432.4138007888746,"sample_count":83656,"score":1435.5926075777552,"score_display":"1436","source_stated_rank":104},"run_fingerprint":"553a5152f73f43ddaa477835b617f815abe31949d13fbacb4bdb673a01e5465d","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_32a0028a60d4548c507548de","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3b7858a50491ace0384325c8e9610433726c9fac41bf11f7a4ca0e7c37107197","metric_details":{"category_scores":{"Agentic Coding":50.25266666666667,"Coding":76.0715,"Data Analysis":78.16333333333334,"IF":61.77074999999999,"Language":79.809,"Mathematics":93.166,"Reasoning":83.2115}},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.6349642857143,"score_display":"74.63","source_stated_rank":40},"run_fingerprint":"fec2f63c16201a10c04cec604fe49bbc1105f743cc75140b4d4f9c7135876563","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":293,"at":"2025-12-15T23:38:26Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-15T23:38:26Z","status":"stale"},"measurement_id":"9be0bd3ec233ac58fff4df0c21eec2eb4c3b67c2b63287e99207afdb7ae438ff","metric_details":{"calibration_error":45.0,"confidence_interval_half_width":1.76,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=gpt-5.2-2025-12-11; createdAt=2025-12-15T23:38:26Z","source_row_created_at":"2025-12-15T23:38:26Z"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"gpt-5.2-2025-12-11","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"176baf2682da17dee4482d7207465a0bb595a6422cfa5a73bd64ed4b8eb6e1ba","result":{"confidence_high":29.560000000000002,"confidence_low":26.04,"sample_count":2500,"score":27.8,"score_display":"27.80","source_stated_rank":10},"run_fingerprint":"c90a6a77d59610535a104cba523e0704faa522a3dac4b68800a978496d90bc9e","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=gpt-5.2-2025-12-11; createdAt=2025-12-15T23:38:26Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_86dae51067db1dc1fc5309bb","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":298,"at":"2025-12-11","basis":"evaluation_at","evaluated_at":"2025-12-11","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"089792bd8ea3358b1fa796120daf5c489e7871fdeab1796f785736889a055c4a","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/openai-gpt-5-2","source_model":"GPT-5.2"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cca07075250f2a205c58921f2b2c27f8e51ef4bbb3905ddae84662322800ab19","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.9,"score_display":"29.9","source_stated_rank":26},"run_fingerprint":"faf98c97786138960893fd673171b13219d160930662ba5c32842ad0bcb2da7a","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f4a4dc1365efaea7b6273a8e","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:27:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:27:55.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4a2f855a6b6f6ccea3a2b74dac9474f2b7522f373643600da323c9f4b696c09d","metric_details":{"best_score_across_scorers":"0.371","model_release_date":"2025-12-11","stderr":1.5283736211823096},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"E3325ytkSNYP8ez53VA4fE","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FE3325ytkSNYP8ez53VA4fE.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/E3325ytkSNYP8ez53VA4fE.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"222ed766bfebee500cdc2d8668f454b6ce0017d1a7ab2bb2f239ab09339d5374","result":{"confidence_high":40.09561229751733,"confidence_low":34.104387702482676,"sample_count":1000,"score":37.1,"score_display":"37.1","source_stated_rank":48},"run_fingerprint":"9ad9007d24decb43019c38c90d46b091fe6cc5a1d72e8df6233ecec187781c08","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:28:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:28:37.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"75bc08e2c890f1af26a1115eb3d2d188e8f4420c33bfefa0023f21f2b0981d0a","metric_details":{"best_score_across_scorers":"0.343","model_release_date":"2025-12-11","stderr":1.5019206922357007},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Xp4qkZ32zjDeSYii2eQpnm","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXp4qkZ32zjDeSYii2eQpnm.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Xp4qkZ32zjDeSYii2eQpnm.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"74284fe748a1a95a90c96d308191a998fb1903e8d137ca95f63a4494ec73cdd6","result":{"confidence_high":37.243764556781976,"confidence_low":31.356235443218033,"sample_count":1000,"score":34.300000000000004,"score_display":"34.3","source_stated_rank":52},"run_fingerprint":"226a59f22bbcb2c3142f1809aef225b09dd4ac9160246e1d9fbcc1590c893c5d","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:29:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:29:07.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f60c22dbd6d92a03ee7349605925158b4328844ed2a9468931861cecd8b8c786","metric_details":{"best_score_across_scorers":"0.327","model_release_date":"2025-12-11","stderr":1.4842213153411161},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"esr5dcSuHiVoteo6vZfHjP","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fesr5dcSuHiVoteo6vZfHjP.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/esr5dcSuHiVoteo6vZfHjP.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"8514e16ea570c16240e6faedec71e6f30a42743c4ef7ee61d7f0229abf73aa94","result":{"confidence_high":35.60907377806859,"confidence_low":29.790926221931414,"sample_count":1000,"score":32.7,"score_display":"32.7","source_stated_rank":62},"run_fingerprint":"2dc596b77d959a42ae6a6da7ab687c31a3396bda61b8515c919af6f5e5665c1e","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:29:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:29:12.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"24c23b07cd397c9859b86528100262f8498f77931671696389f0fc3c89ad80d8","metric_details":{"best_score_across_scorers":"0.328","model_release_date":"2025-12-11","stderr":1.4853842487270312},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Cyw3GA9a64AmT4pqLAhtTF","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCyw3GA9a64AmT4pqLAhtTF.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Cyw3GA9a64AmT4pqLAhtTF.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6e0c7325aac846be22a8185c00ce9c3b016f2918aba81c35e59eaf04d37d40f1","result":{"confidence_high":35.711353127504985,"confidence_low":29.888646872495023,"sample_count":1000,"score":32.800000000000004,"score_display":"32.8","source_stated_rank":60},"run_fingerprint":"fd3d3e10be9a710ebf791ead3a5747bc4bcbd709f7f46eb34ab44e48cdfea043","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40883fb2b66729f88d4e7077","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"670de83994ed4ac2c20ba286c71bffaf74688e54322b457927dd2d30f2e1273d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.149037,"date_is_proxy":true,"latency_seconds":173.214,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.837},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":91.919,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"xhigh","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.2-2025-12-11","zero_shot_accuracy":91.414},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"0e53c52812ba5fb6dfa5862b33dba492022dfa6bba8d69a7933d407ab461e32f","result":{"confidence_high":95.26652,"confidence_low":88.06548,"sample_count":396,"score":91.666,"score_display":"91.67","source_stated_rank":18},"run_fingerprint":"fc5beacaae9b9fe42d0f3721a8097e94bb5512dd9a1f8db7f691f0dbd7b6c262","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":296,"at":"2025-12-13T09:04:00.000Z","basis":"evaluation_started_at","evaluated_at":"2025-12-13T09:04:00.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5fbefad8f470a3d1659f10e4e9bf439dae2f7df7e30364fe9ef2ae084b9f41cb","metric_details":{"best_score_across_scorers":"0.914","model_release_date":"2025-12-11","stderr":1.7999999999999998},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Ejo6Fy7XYxUDcCwbejfcYX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEjo6Fy7XYxUDcCwbejfcYX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Ejo6Fy7XYxUDcCwbejfcYX.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"1d3f22c369d85f45b7538d1c66a6edbaa7a06c4a4ce29b35e52c83cc1c89b9eb","result":{"confidence_high":94.92800000000001,"confidence_low":87.872,"sample_count":null,"score":91.4,"score_display":"91.4","source_stated_rank":31},"run_fingerprint":"3064d93a0fb85e628007cd8d43f77a135062c368ef4cc2b2ae4e48c86e72783d","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":297,"at":"2025-12-11T20:49:22.808Z","basis":"evaluation_started_at","evaluated_at":"2025-12-11T20:49:22.808Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"78940ebfc228bf7acb629cbc5e3cf58103cb0b4128ca2e78f68199cc53796b06","metric_details":{"best_score_across_scorers":"0.8819444444444444","model_release_date":"2025-12-11","stderr":1.9138582776339241},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"3Q6osT8EYezayG7yfxkSpr","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3Q6osT8EYezayG7yfxkSpr.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/3Q6osT8EYezayG7yfxkSpr.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"c73fb1620b920151253e3c9858471c8ce1bdc0dd7a538b5d9005fc17c2866a8f","result":{"confidence_high":91.94560666860693,"confidence_low":84.44328222028196,"sample_count":null,"score":88.19444444444444,"score_display":"88.2","source_stated_rank":62},"run_fingerprint":"e727ba58f688b7f19f7b43e2563d02fab5ccfaefec3224f349cf35aadf397a11","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":297,"at":"2025-12-11T20:49:17.803Z","basis":"evaluation_started_at","evaluated_at":"2025-12-11T20:49:17.803Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f615c40b37036c1d831ae015d83b150d954bf25c6466ad131323025e96be524a","metric_details":{"best_score_across_scorers":"0.8787878787878788","model_release_date":"2025-12-11","stderr":1.9090689343231744},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"MuhtxxvALeZZ37cSG5CVcc","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMuhtxxvALeZZ37cSG5CVcc.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/MuhtxxvALeZZ37cSG5CVcc.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d0683760467d70e9fe91f6d99790989fd3bb9979034a668111cbeeb2be694e86","result":{"confidence_high":91.6205629900613,"confidence_low":84.13701276751445,"sample_count":null,"score":87.87878787878788,"score_display":"87.9","source_stated_rank":63},"run_fingerprint":"b357d69459e04ec068b8a89b9c064b4f0674594e7b7826e580da836160255739","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":297,"at":"2025-12-11T20:25:14.724Z","basis":"evaluation_started_at","evaluated_at":"2025-12-11T20:25:14.724Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e94bb49d80e8ba39a68233eaaad7e1a165dc1df18086a07afae3c5499bf55098","metric_details":{"best_score_across_scorers":"0.827020202020202","model_release_date":"2025-12-11","stderr":2.265112408975899},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"oD4Jpoi75KWChqTPDaN9aB","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoD4Jpoi75KWChqTPDaN9aB.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/oD4Jpoi75KWChqTPDaN9aB.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"f1e9b7b49ac94928d0a99349a727b66c61d78083c87c59ff9061823c9bd144ae","result":{"confidence_high":87.14164052361295,"confidence_low":78.26239988042744,"sample_count":null,"score":82.7020202020202,"score_display":"82.7","source_stated_rank":112},"run_fingerprint":"fecb2c6400dcac733a04f545bedfb481240cf3b6222e4e4984bce4b68fab277b","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40883fb2b66729f88d4e7077","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":83,"at":"2026-07-13T17:28:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T17:28:07.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1ac5f7555991b25eb7ab775caa479d23d8a92b79f2adc7ebca02b16d5bf0f573","metric_details":{"best_score_across_scorers":"0.7323232323232324","model_release_date":"2025-12-11","stderr":3.1544498882702823},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4sPs53wmM9bvN8JXzsiNCk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4sPs53wmM9bvN8JXzsiNCk.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4sPs53wmM9bvN8JXzsiNCk.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"2f02bec6e46e623ea27d8b46d0435f9103a231ba1bd75ef36635e01b7490938f","result":{"confidence_high":79.41504501333299,"confidence_low":67.04960145131349,"sample_count":null,"score":73.23232323232324,"score_display":"73.2","source_stated_rank":167},"run_fingerprint":"4c4ef9efb6750114a0c90175ad16f86b5e87fbf9e942de1febc95204606e04aa","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_ff3aeab8b13f468a67900233","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"71100c3b250b9602503d0adb3004ffb76b26aeeadc11b3c8e5638e1dd4441924","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.03698,"date_is_proxy":true,"latency_seconds":35.413,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.344},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.2-2025-12-11"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e4e585f31b396b6ced770c725a3cc2134feff33bc8c40588a8aa2dab55a9f267","result":{"confidence_high":86.90823999999999,"confidence_low":85.55976,"sample_count":null,"score":86.234,"score_display":"86.2","source_stated_rank":46},"run_fingerprint":"6aecdcdab8085d97e7238fb975f2a1b4532400a3f6366c3d9b2da59887217352","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":295,"at":"2025-12-13T15:56:00.000Z","basis":"evaluation_started_at","evaluated_at":"2025-12-13T15:56:00.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"39822e7c8df3f2793c2c3ec695b7e77ea0724ad19132e02449d4deeaf62f18ee","metric_details":{"best_score_across_scorers":"0.961","model_release_date":"2025-12-11","stderr":2.7},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"R393BZjXtkEYju29gs5R8B","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FR393BZjXtkEYju29gs5R8B.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/R393BZjXtkEYju29gs5R8B.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"262d18a12360b932973b89b9d03963035c3c57ce099ed5777db7c6d87332a7f1","result":{"confidence_high":100.0,"confidence_low":90.80799999999999,"sample_count":45,"score":96.1,"score_display":"96.1","source_stated_rank":36},"run_fingerprint":"805d6a3b8b329ede157c5911124ebc0351e88133416de85936bdf298667b2dde","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":297,"at":"2025-12-11T20:49:15.912Z","basis":"evaluation_started_at","evaluated_at":"2025-12-11T20:49:15.912Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"cdb64a0651a31986fda2786c103e09fd101b3cddacba232fec1bc70ee4a2ccbd","metric_details":{"best_score_across_scorers":"0.9611111111111111","model_release_date":"2025-12-11","stderr":2.5997226390952677},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LiUMZyNSM8q93KDubWgYCh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLiUMZyNSM8q93KDubWgYCh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/LiUMZyNSM8q93KDubWgYCh.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ba4c13ed76ee0f51cf276664d7c6f0a4cf81d679144fda4cc2d33f7a217a6058","result":{"confidence_high":100.0,"confidence_low":91.01565473848439,"sample_count":45,"score":96.11111111111111,"score_display":"96.1","source_stated_rank":34},"run_fingerprint":"3c126a005b5e7ab92c0c0f3f15f1d4b3782628a9ce20ee2a70dc1cbbc29f1156","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":297,"at":"2025-12-11T20:49:22.447Z","basis":"evaluation_started_at","evaluated_at":"2025-12-11T20:49:22.447Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"34e9d5c47a563931b0e02d665624b1deb799e6c84c1f94e177d8b85581e3ead2","metric_details":{"best_score_across_scorers":"0.9388888888888889","model_release_date":"2025-12-11","stderr":3.0675842208062716},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Z2ZwwqLDDrBrEYhtKtFLCQ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ2ZwwqLDDrBrEYhtKtFLCQ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Z2ZwwqLDDrBrEYhtKtFLCQ.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"49b82c5a0a7d20eaf02dab12977c605f1a84f5e7c6f462501615eae902059664","result":{"confidence_high":99.90135396166917,"confidence_low":87.8764238161086,"sample_count":45,"score":93.88888888888889,"score_display":"93.9","source_stated_rank":51},"run_fingerprint":"3d03fb372c049e87ec8dfe8c580089c3425a5b261aa0ead7198170e2beee8c6a","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":297,"at":"2025-12-11T20:24:25.981Z","basis":"evaluation_started_at","evaluated_at":"2025-12-11T20:24:25.981Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"2ef369386d1da618f3fa1aa9e77e85074df0a7876a851cee17a32f77b72d6033","metric_details":{"best_score_across_scorers":"0.7888888888888889","model_release_date":"2025-12-11","stderr":5.006659426035015},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"5Hcw3Td96qF66quWrDMKJB","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5Hcw3Td96qF66quWrDMKJB.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/5Hcw3Td96qF66quWrDMKJB.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"68b9e918eef0c3f63efad1de2ee71e7925160a8a9ac5e761eead440c4fdfe48c","result":{"confidence_high":88.70194136391751,"confidence_low":69.07583641386026,"sample_count":45,"score":78.88888888888889,"score_display":"78.9","source_stated_rank":116},"run_fingerprint":"bab9aa0515b057afa5450816de47109769859847267a6469a1e50ced200f0b5a","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40883fb2b66729f88d4e7077","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T17:28:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T17:28:07.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5c7db48d858d30939cf6c2ee4ddd24d2fbf5f87eeef9e28f0e8c24e7cc923a1b","metric_details":{"best_score_across_scorers":"0.6222222222222222","model_release_date":"2025-12-11","stderr":7.309112127323451},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"UBKiF8QVUmHwqgx4rF7CDA","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUBKiF8QVUmHwqgx4rF7CDA.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/UBKiF8QVUmHwqgx4rF7CDA.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f5966987a8fa1e0747ca6d7d05888fcd0a05d9d0a297a3ddefdd094434baebc2","result":{"confidence_high":76.54808199177619,"confidence_low":47.89636245266826,"sample_count":45,"score":62.22222222222222,"score_display":"62.2","source_stated_rank":160},"run_fingerprint":"d10a31b6e5a454b701709ac59d27c985439d5ad03d0f972456ed76118f887c27","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_ff3aeab8b13f468a67900233","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-11T02:48:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T02:48:30.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"d6d952006738277c081c36315eb47f7af8a95ba3d22e03806f47a83a511b4fe0","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.674","leaderboard_private_problem_count":285,"model_release_date":"2025-12-11","public_example_count":10,"stderr":2.8000000000000003},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"PeZ84GTQdSYQzDxRMWA2Ha","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.2 (xhigh)","thinking":null,"upstream_model_id":"gpt-5.2-2025-12-11_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"5686c6437aca0367bc365c1104f9271197e9b9dab735a9cea4972f021a45bc40","result":{"confidence_high":72.888,"confidence_low":61.912000000000006,"sample_count":285,"score":67.4,"score_display":"67.4","source_stated_rank":27},"run_fingerprint":"6b071ade0550a8e680d59d89babc0c8b32a7e7c5d9ebb714026393195b86a857","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"18e79b51366dbe3dc5cbf1c49fc0175818469f1bc124e3662ab64acc1a8c0c2f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.8982,"model_release_date":"2025-12-11T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-2-2025-12-11-thinking","model_type":"CoT","upstream_model_id":"gpt-5-2-2025-12-11-thinking-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d01d54cf6663c2478759bd86f00cfc3e917f8de7551571b190f30e5bbafc1a62","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.910000000000004,"score_display":"53","source_stated_rank":92},"run_fingerprint":"594484548a1a61e1e741675c69679fb33d77620461174b5e49601ad63356d56d","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"008994760dc5273616cae8ebc3ca2ca20d51ba049d7f016b2fba6440886ea8fb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.393,"model_release_date":"2025-12-11T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-2-2025-12-11-thinking","model_type":"CoT","upstream_model_id":"gpt-5-2-2025-12-11-thinking-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ebe9fb1dd6638fb5617cb61b9dd76e7cdb4fc0e973afc65b0b179a71cb60ef38","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.33,"score_display":"43","source_stated_rank":101},"run_fingerprint":"ff2d77f89bf76f8865043ce0d9d2d458ee85529de38a88794f5eed407b84077a","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6537a8a43edb48e310c93f9426848616e4f36885afb39ad9e47dd26d33bb710f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.7594,"model_release_date":"2025-12-11T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-2-2025-12-11-thinking","model_type":"CoT","upstream_model_id":"gpt-5-2-2025-12-11-thinking-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1f82cb425b0255652eb2083ea33f284feb0c07bfa8445c28cbbb6c268f577b58","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":26.669999999999998,"score_display":"27","source_stated_rank":125},"run_fingerprint":"964897f834c3301cde0f88dedd4c5996f9848acf281d07bd4b43452adfef502c","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"442b814118a3d0335f26796f3898fb3ca608a98cd0cc7d2bf95cddf08d92398a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.2642,"model_release_date":"2025-12-11T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-2-2025-12-11-thinking","model_type":"CoT","upstream_model_id":"gpt-5-2-2025-12-11-thinking-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"06f26e0dda043b522e9b358c84c52909a7b99657cf1f23c125882d0bdf4d3ad0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":9.719999999999999,"score_display":"10","source_stated_rank":145},"run_fingerprint":"450f03f346a773d9f71bc3efa9b6d6b54c8a224408abd0dfc5806220623f9681","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40883fb2b66729f88d4e7077","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6e04448c419ffca2ab1f666febe21e5f2ba504d291a9e94fcdad67ffd2845619","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0818,"model_release_date":"2025-12-11T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"","model_type":"Base LLM","upstream_model_id":"gpt-5-2-2025-12-11-thinking-none"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c3d58a3af8073f304c00ebbae73e4ca1cfc18b86bcefbb409628a2facfcf245d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.83,"score_display":"1","source_stated_rank":229},"run_fingerprint":"828ce1ea58efe49e44c82ba29bd2ff58aeae96e548d716f949c9b090dd1937a4","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_05dc6ab7700ac47b0c25a3a0","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ed107afe63aa585cb5a21c258edc0b998b068e7699b354cfd2443350a881e21d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.8982","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (X-High)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f81eed7a33a4384c17c167e8c91003ccc2e914f2f955f87a3c0a542c10374e80","result":{"confidence_high":58.008372,"confidence_low":47.750178,"sample_count":360,"score":52.910000000000004,"score_display":"52.9","source_stated_rank":93},"run_fingerprint":"4a72c250ed76579a4f1a4e27c225682cd0b9cd49163f350268d9150204700e64","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0822ae96c17732284ee91b23520fe42834af80ebf5a8563496c44cd586f31b65","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (xHigh)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"73486c48514be4365c2a40c78ffe6056d4976f84d87c5377cf98ba3497078a5d","result":{"confidence_high":58.008372,"confidence_low":47.750178,"sample_count":360,"score":52.910000000000004,"score_display":"52.9","source_stated_rank":93},"run_fingerprint":"865a678fbc2afd6b5e1e01ed2f5896ad515e4e96af7a92c6c3e65f56578a58d2","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1a822c95a5ad3bb591eae512464b63841bcbb78867e87748693d43e77b19ac76","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (XHigh)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5e4e2def98963f5c30bcf92e55713149c012e055e12898e7251e306ffa89233b","result":{"confidence_high":58.008372,"confidence_low":47.750178,"sample_count":360,"score":52.910000000000004,"score_display":"52.9","source_stated_rank":93},"run_fingerprint":"ee16bc46d24a76fe3b9a7a4fbf6ee84958fa14b5c3c36b4dc2c21345f1d9d266","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"345e9c5da6c9b833669c806258cb5c84cca2e5d502e127e8651dd4ced1877a40","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.393","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (High)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b303acc28525477abc772a419e83ff8981e9abc72bc3c394eb16a8afc7caff0e","result":{"confidence_high":48.492706,"confidence_low":38.308144,"sample_count":360,"score":43.33,"score_display":"43.3","source_stated_rank":103},"run_fingerprint":"2098fb8f7c3277846df3ccf63531e8cf1a6ae3d157a4e0f2502e33681cc19e2c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"98114578805f4f87324b75d8212ae583eff6105e99023196c2c6861f5766ad8a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.7594","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (Medium)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b5de12648bffc3e714f6a77810003619c8d4c24b45bd1a45e43ad7b81e462838","result":{"confidence_high":31.467146,"confidence_low":22.365511,"sample_count":360,"score":26.669999999999998,"score_display":"26.7","source_stated_rank":122},"run_fingerprint":"8092e7cb0a3ae491bb35e1404f2ca80c366bcf6964aefec51c8278c7a5eb7b1c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1ccd34af446e20ec2adfcaf8a88f58439c7497c956b7eb8599ed26eb6e481275","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.2642","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (Low)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ccaee9d4a315072ae456793ef0bfb075ad2dd4c5e5d4f8562da9ba2ddd4a56f9","result":{"confidence_high":13.21875,"confidence_low":7.071837,"sample_count":360,"score":9.719999999999999,"score_display":"9.7","source_stated_rank":141},"run_fingerprint":"15c35af6bfc618c06abe5c83e779578061137b64e8aa227ecbca84671da4ea22","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40883fb2b66729f88d4e7077","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"62c2718b70531b29d4b0ae66aed7774e0bc8847fd30d2daf798d294df3323b1f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"cd02d6a8b9b86c5b39e5459c43253a4a4844bdbbf4bdfeec52e64bfed358f0ba","result":{"confidence_high":2.416212,"confidence_low":0.282105,"sample_count":360,"score":0.83,"score_display":"0.8","source_stated_rank":220},"run_fingerprint":"81b7a44159fabd76341ce0cc542c88ecd505e706e9de9ed757085f983642716c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a54554c659d84ee3c53c4baf","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T14:48:04.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T14:48:04.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"2d3a2f14695e062b8b98130295cb2c3c126f6cdf0e9d981c5d2910b9618ddbdc","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.317","leaderboard_private_problem_count":41,"model_release_date":"2025-12-11","public_example_count":2,"stderr":7.3},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"J3NUxsK2GXq9Whkjn7mEVm","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.2 (xhigh)","thinking":null,"upstream_model_id":"gpt-5.2-2025-12-11_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"549bfdf41fd03bd148e9ca762423d3d6b133ad71b87a65916b717b0b26fa4850","result":{"confidence_high":46.007999999999996,"confidence_low":17.392,"sample_count":41,"score":31.7,"score_display":"31.7","source_stated_rank":33},"run_fingerprint":"9e0db873567d602d4bf585008da3f56977da27122320212d49747eb640824217","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6dc60c2169a8cbb234a19d4d1483636373990c2b824fc414ef3f5f23ebdf84ff","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.061121,"date_is_proxy":true,"latency_seconds":96.324,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.822},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.2-2025-12-11"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"09147cd9831f99dec3a12f8c8c01a4f3f31dc61a9c373701232e68d8b7119a1a","result":{"confidence_high":88.27812,"confidence_low":85.05588,"sample_count":null,"score":86.667,"score_display":"86.7","source_stated_rank":17},"run_fingerprint":"ea77448b9096349f64b92868547b8d406a874e35946fc4f2db20b1f94739fff2","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"298bc186b13ceae476a5abcad069ffbc1bfbad957254e97efb63beb28de1cc18","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_high","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"3f440b29a11b0bbf50a91dff1d673b2869ac88c6ad8dd8682cb0a9fcfe3be965","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.800000000000004,"score_display":"45.8","source_stated_rank":62},"run_fingerprint":"26de772d46f240203cdee82b450d662f9463c87a83e2178ab503ff2a8eaf8d75","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1717c900a8432b307520579dea05385d74d8c541dcd0300b5fcdd7a758b6a09a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"d805f3443668e0c4ab7e0ed704f1f94d51d6eef792093a4edb18e3db1233ab32","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.8,"score_display":"45.8","source_stated_rank":63},"run_fingerprint":"892dd7e78558590283cb91686400b46be1d0d27584c33e4fe9107b2858cd7079","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_070939b181f9626ef3ce6fe3","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":294,"at":"2025-12-15T10:01:10.174Z","basis":"evaluation_started_at","evaluated_at":"2025-12-15T10:01:10.174Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3740e369f37e726223c9d5ec4649d94128a08601b0a6a7577ce32e024119c3ac","metric_details":{"best_score_across_scorers":"0.49","model_release_date":"2025-12-11","stderr":5.0},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"manual_gpt_5.2_xhigh_chess_puzzles_frankenstein","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"cbce21016a5eb11ef96989cb4533a18eab979343a922bdbbf414470eefbdc24a","result":{"confidence_high":58.8,"confidence_low":39.2,"sample_count":100,"score":49.0,"score_display":"49.0","source_stated_rank":12},"run_fingerprint":"f10c931449309729c2206cbc9eb32d11ae8c61916151e886e1074343bd386408","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":297,"at":"2025-12-11T20:47:46.679Z","basis":"evaluation_started_at","evaluated_at":"2025-12-11T20:47:46.679Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"6d5e7fb3fee3095497c269e2e02bf7a5c6154fa5fb8ba02fe51a2735476de374","metric_details":{"best_score_across_scorers":"0.4","model_release_date":"2025-12-11","stderr":4.92365963917331},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"R3bUSJCQiuGjg6TzBo7k8Q","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FR3bUSJCQiuGjg6TzBo7k8Q.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/R3bUSJCQiuGjg6TzBo7k8Q.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1a4acefbadea671a80cbb23e8ce518bf528290833c44c8058e5f2d169efdea11","result":{"confidence_high":49.650372892779686,"confidence_low":30.349627107220314,"sample_count":100,"score":40.0,"score_display":"40.0","source_stated_rank":24},"run_fingerprint":"a773951a02c64606b589daba2dd5f275e7720f583974806e239edea73152ceb4","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":297,"at":"2025-12-11T20:47:46.894Z","basis":"evaluation_started_at","evaluated_at":"2025-12-11T20:47:46.894Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"cda7f167fc7c624e1ee6c8628f9bd19f8022151bab0de8afd20d434a3a6633e3","metric_details":{"best_score_across_scorers":"0.4","model_release_date":"2025-12-11","stderr":4.92365963917331},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"fCzTfk9GNPokGEVSg2ry4s","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfCzTfk9GNPokGEVSg2ry4s.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/fCzTfk9GNPokGEVSg2ry4s.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"53b56c728d68e2a5f1634fe5b0e4468974dc4a79779e890ee3a294e5d212495c","result":{"confidence_high":49.650372892779686,"confidence_low":30.349627107220314,"sample_count":100,"score":40.0,"score_display":"40.0","source_stated_rank":24},"run_fingerprint":"600b9c9cc8863b643a26890d34ea474f461f772710ad861258ec5fd903320e76","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":297,"at":"2025-12-11T20:24:27.647Z","basis":"evaluation_started_at","evaluated_at":"2025-12-11T20:24:27.647Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8e41cf40445165dcdebf81c14a55bd92c7f3471682005f38642cd6da5f157fcd","metric_details":{"best_score_across_scorers":"0.23","model_release_date":"2025-12-11","stderr":4.229525846816508},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"SUBFSiYKfaantxswKig2a2","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSUBFSiYKfaantxswKig2a2.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/SUBFSiYKfaantxswKig2a2.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ba2f0be801d3dbc0b8a0e00055f0ef9e2e9d5133a7adad95662ef12e68aad87e","result":{"confidence_high":31.289870659760354,"confidence_low":14.710129340239645,"sample_count":100,"score":23.0,"score_display":"23.0","source_stated_rank":74},"run_fingerprint":"1b701b23fe227ff8f948433c263ee2fbbe1cfad5538eeac82568b11f38dab6f8","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40883fb2b66729f88d4e7077","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T17:28:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T17:28:07.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8987c535cc2b4bc88899e656fac660dd36fe4cffb506dcdc667bd423cd7fe617","metric_details":{"best_score_across_scorers":"0.04","model_release_date":"2025-12-11","stderr":1.969463855669324},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jMRYcMRgjaXW3N6q2JogNW","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjMRYcMRgjaXW3N6q2JogNW.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/jMRYcMRgjaXW3N6q2JogNW.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8ce43d6b73e122a28fa6c61cb991bac05d5ac34ab77aa8644caaf8fd1521ee72","result":{"confidence_high":7.860149157111875,"confidence_low":0.1398508428881251,"sample_count":100,"score":4.0,"score_display":"4.0","source_stated_rank":169},"run_fingerprint":"33313e7f908423c0675d2dccd3e2ab43ee29cb1af2d41fba66f13ec8289d0119","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_ff3aeab8b13f468a67900233","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"63e15cbe1756210543ffbdb73eba816efe23ca49374bbff111a90a9f9fb6834e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.96","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (X-High)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5093acc4b109689bb8f30a930c66e72ace881fd78b814418150bdf331bc1ef81","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.2,"score_display":"86.2","source_stated_rank":95},"run_fingerprint":"29e82abe7b646a763769e3123cf772dd85dbc54800e724bd49c12e753884c4f6","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"81c08de67994a6c909c146247642eaa179f53c7f7f085ad8a971576f26e956ef","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (xHigh)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c0718cd0058b95682d3b80db120f66ae82c246b66569f63d3983ff35b7c06c45","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.17,"score_display":"86.2","source_stated_rank":96},"run_fingerprint":"c3c81e741e156053e787067a260edc96502f6db2c87c7363f17ff6c243b66e24","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ad10f172f212d2059172cc9e75fc5b858ac1b07f72558509fdd3e29e3e722e0a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (XHigh)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"37b0b157813da8097c2a73b9e849142f62299612fcd33104e757d3bacd9d65bd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.17,"score_display":"86.2","source_stated_rank":96},"run_fingerprint":"d1f92dc57006fb143acafd1465a3d6c4b5200d077310dfe10c718735e7541bf5","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e69eda771ad41789acf634e67fe16d2d96d75c941097a2474930d7cf3e32a809","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.5196","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (High)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b9c8ed3ea2dbf0b361a6870a3fe3decc80d907429b847f451070834547f669c5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.67,"score_display":"78.7","source_stated_rank":115},"run_fingerprint":"e4b22f39e0aec24875b56496978d18ea53fa6bff6898621c608af0afe5d0a1ab","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"124dc5a6c2bc5abb3c56f8a1f969692361b03a87fff98d11c253f475776fd2c2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.3448","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (Medium)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"186afdd3bd4d628c030d4355fcf714fc841d11ac157f2ea671ff1e693eda572b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.67,"score_display":"72.7","source_stated_rank":131},"run_fingerprint":"76637f2fe70fa883c83b957f10c2fa883260e235f441f5b3527d03533466ff4c","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"44e8c63a24ba0352acc8ec199290495209ee754f3b4c0f00e08ede9ded4e2a7c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.1657","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (Low)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a1ae526d3e5c30df0cb7d998469df463eb197a65b890fc2e9d5398a175eedb36","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.669999999999995,"score_display":"55.7","source_stated_rank":161},"run_fingerprint":"6f18330957d5e6aa2eddf23f989b127af2083e41c8f56b11b32179e459bf0864","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40883fb2b66729f88d4e7077","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"79b099f2fefc02c70a256d20f06096d2592c7f44a4e88306b28a1ae86daeeccd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3beecf2a96352a81b857e8f4a2b85e8751a0ae2896bc963a070e01b2f246c176","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":12.33,"score_display":"12.3","source_stated_rank":245},"run_fingerprint":"16dd0c04ab071106c0af42e180679bf6f017def80d024be5f14161e1f0f77a92","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a6f4130ca21f88e8d4d0937e","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7b500107fd7782ae44e5e3d91d0740e88860c79c769856e9348dec57a1a1a539","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.082","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e4c2c7a57a7c1b5708291512bab760d20676fb7685035a9694a7824e011a523e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":12.33,"score_display":"12.3","source_stated_rank":245},"run_fingerprint":"b878b8b89d3d02a6bf648770b162b71ad84a0a44a8077710fc66f8900b2082c7","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_ff3aeab8b13f468a67900233","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"08c01ca87d6918cfeaf7e8cffb85a10daaf88f9988c95588315aa24f615cf5b0","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.453372,"date_is_proxy":true,"latency_seconds":640.472,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.917},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.2-2025-12-11"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"945fd6ffb0f213f3291ec8755102e9409111131c3df00c2c70c3f55ca39717f8","result":{"confidence_high":79.55732,"confidence_low":72.04267999999999,"sample_count":500,"score":75.8,"score_display":"75.8","source_stated_rank":43},"run_fingerprint":"fd6b3b6968eea2160be21f8250a537446eb84e853a7f48bddb881f80557e1c6a","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":230,"at":"2026-02-17","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17","status":"stale"},"measurement_id":"3955c441b5571936cdb20dafb7977a0ca91c38c6801d14733d61606aa6f9e602","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_gpt-5-2-high","logs":null,"model_display":"GPT 5.2","model_org":"OpenAI","reasoning_effort":"high","resolved":72.8,"site":"https://platform.openai.com","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: gpt-5-2","Org: OpenAI","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_gpt-5-2-high/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"000ff9060f41559525fe3989e18274663aa13fd9feed1c750ec365221dbcb084","result":{"confidence_high":76.515656,"confidence_low":68.736662,"sample_count":500,"score":72.8,"score_display":"72.8","source_stated_rank":17},"run_fingerprint":"0f5ca2dc862f2c6425195296f612190c94af308d18c0d1d2d1e264129ebe5c75","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":298,"at":"2025-12-11","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-11","status":"stale"},"measurement_id":"5f3e916229e5ae9c2f0047c42e00dc70468024fcb838c38b7a4764ce061eaa5a","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2025-12-11","folder":"20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high","logs":"s3://swe-bench-submissions/bash-only/20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high/logs","model_display":"GPT 5.2","model_org":"OpenAI","reasoning_effort":"high","resolved":71.8,"site":"https://platform.openai.com/docs/models/gpt-5","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: gpt-5.2-2025-12-11","Org: OpenAI","System: Attempts - 1","Mini: 1.17.2"],"trajs":"s3://swe-bench-submissions/bash-only/20251211_mini-v1.17.2_gpt-5.2-2025-12-11-high/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"40d6281107634ca1196c35e40aa90a79b10f21443bbd8dd30ea405b916c50dbc","result":{"confidence_high":75.566422,"confidence_low":67.701144,"sample_count":500,"score":71.8,"score_display":"71.8","source_stated_rank":21},"run_fingerprint":"cf9466685cccf5597b5814729718b846cc36079a1773539ed7d28adf21ad3e9f","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":298,"at":"2025-12-11","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-11","status":"stale"},"measurement_id":"fb23892f408327905e5fc9728097db40209f042cda6071e64397deaca9ac10ec","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2025-12-11","folder":"20251211_mini-v1.17.2_gpt-5.2-2025-12-11","logs":"s3://swe-bench-submissions/bash-only/20251211_mini-v1.17.2_gpt-5.2-2025-12-11/logs","model_display":"GPT 5.2","model_org":"OpenAI","reasoning_effort":null,"resolved":69.0,"site":"https://platform.openai.com/docs/models/","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: gpt-5.2-2025-12-11","Org: OpenAI","System: Attempts - 1","Mini: 1.17.2"],"trajs":"s3://swe-bench-submissions/bash-only/20251211_mini-v1.17.2_gpt-5.2-2025-12-11/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"b3f1e5d33cbef34ccb85ae961dc5527346df98e3df0cc5d3f10ae4b79b7e63e6","result":{"confidence_high":72.896177,"confidence_low":64.814087,"sample_count":500,"score":69.0,"score_display":"69.0","source_stated_rank":34},"run_fingerprint":"6c39d2c327c31149bdbc5231c681306f1321354a0fa0c2c9571d909f462d4926","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_181e8d680b21e662e4cf3e90","provider_config":{"source_id":"official-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":235,"at":"2026-02-12T13:34:00.865Z","basis":"evaluation_started_at","evaluated_at":"2026-02-12T13:34:00.865Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"046d50752f7d64ca8b84598a352a38391e56a3da168d0aea42c7a718b59fafab","metric_details":{"best_score_across_scorers":"0.737603305785124","model_release_date":"2025-12-11","stderr":2.001783206492828},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"9owdbpnWJ5pkQQC3GWKP5L","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9owdbpnWJ5pkQQC3GWKP5L.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/9owdbpnWJ5pkQQC3GWKP5L.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"04a81f3feab5d0c95f50ce42416b677ad8640a8670a524eb5600ee0f439989b8","result":{"confidence_high":77.68382566323834,"confidence_low":69.83683549378645,"sample_count":484,"score":73.7603305785124,"score_display":"73.8","source_stated_rank":18},"run_fingerprint":"10d8a6818eadd74704c31a769d22b9369fcde89013c415e55165f3fcd6b70189","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b5f5937b2cb1e2b756288948","split_or_window":"2025-12-15/2026-03-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":218,"at":"2026-03-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"d15355f780b41cb618fbf22675a85d380bb951e86e22e13702f1430428ddf0d5","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-12-11","pass_at_5":68.29268292682927,"potential_contamination":false,"sem":1.2479931591919553},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"gpt-5.2-2025-12-11-medium__tools","window_from":"2025-12-15","window_status":"historical-post-release","window_to":"2026-03-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"f288398b66ed709eb0d9a928ab5ecc2b376564d2d163f6ecd2518791219528c7","result":{"confidence_high":61.46133651710404,"confidence_low":56.56920333307158,"sample_count":null,"score":59.01526992508781,"score_display":"59.02","source_stated_rank":null},"run_fingerprint":"2d2a0b7e33c974264cd85095cd877b3081a195aaa5b93699a2a941cd99d62dff","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_c32ea17b0523e47d804b902b","split_or_window":"2025-12-15/2026-02-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":246,"at":"2026-02-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f77b01da7bfaef2c1b43bcf47079da7042dab1574170930f0cf272ce89482cd3","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-12-11","pass_at_5":62.121212121212125,"potential_contamination":false,"sem":0.9677562613114045},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"gpt-5.2-2025-12-11-xhigh__tools","window_from":"2025-12-15","window_status":"historical-post-release","window_to":"2026-02-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"afc0c042388820f9a940943d284055d915a7e8c69d9f638fc57b786732938772","result":{"confidence_high":57.82221019757828,"confidence_low":54.02860565323757,"sample_count":null,"score":55.925407925407924,"score_display":"55.93","source_stated_rank":null},"run_fingerprint":"51c57b347d92db09517956b1e1e005568685d82fc2db5c1bb8e8a0c26535d512","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":251,"at":"2026-01-27T03:52:41Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-27T03:52:41Z","status":"stale"},"measurement_id":"1716fe888db4a2b88f6ba39bb208f8c5377e717d9a600bf99a6a62a21540a2a6","metric_details":{"confidence_interval_half_width":2.15,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gpt-5.2; createdAt=2026-01-27T03:52:41Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2026-01-27T03:52:41Z"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-undisclosed-harness","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":false,"protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"gpt-5.2","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576"},"run_count":null,"scaffold":"submission-specific","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"f56371a3895bfb8dafcd40a40784a8435c9b1365ca1887e19c3fe62711166362","result":{"confidence_high":32.09,"confidence_low":27.790000000000003,"sample_count":731,"score":29.94,"score_display":"29.9","source_stated_rank":16},"run_fingerprint":"9f4f78ee8efce25c0bbdf36dce468d2757e18f856bc1208831146f9813cccdb4","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=gpt-5.2; createdAt=2026-01-27T03:52:41Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_be31f1d26694f408fcf20b8f","provider_config":{"source_id":"scale-swe-bench-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-undisclosed-harness","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":291,"at":"2025-12-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-18","status":"stale"},"measurement_id":"f97ce61cc1e63f374d5bcf2a60ed2bbb39d2dfdc92ff7137513981a804fc455b","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_high","source_row_date":"2025-12-18","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"5e521ae728dfdea359144507dde4e6beeafe8b17a3a8ca8e8abbdc88798d0ee9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.450000000000003,"score_display":"27.5","source_stated_rank":10},"run_fingerprint":"6213a10dd5cb3f36d57056ebef1a6319430f836c12d1aaae3957e4995dec07d3","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d14fcba528e5e008c0bfb36090d85a7f3a8295b8fc05a92862bdadc0df2c5af6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.135162,"date_is_proxy":true,"latency_seconds":151.513,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.027},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.2-2025-12-11"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"d3f8cd8466ccff4d5e55f8976ee8b05875e9a10fd872ac34ba8edbb7595d4139","result":{"confidence_high":87.37392,"confidence_low":83.34808000000001,"sample_count":null,"score":85.361,"score_display":"85.4","source_stated_rank":33},"run_fingerprint":"ae154d3352d6ce94329438ce73d3e47f24c2713e6ad73f471124dcb9fdb4c9bc","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6039ef0b1cb06251e4106504f62f9bc723304e8faa091abeaeb14bdbcf3e193b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.296865","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.2-high","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"f0a4cfc02667cf63bd3491e1b679861c8ac630ab58bfea090534396c2f2724dd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1293.55,"score_display":"1293.55","source_stated_rank":30},"run_fingerprint":"1e63975de43da9d5b4019177a43e662aa0150036b31492dba0bafd1da9e82c80","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"85ec26f779ea5b786cca805aea105d486a015c8a52cc230d94acda9f47c767e5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.142947","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.2-medium","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"c3a16dc8f370f7ddd198dd797e62adf9720b321a6430d1ccff755f371e16cb76","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1249.83,"score_display":"1249.83","source_stated_rank":32},"run_fingerprint":"7f09683f312a740e696fcbe3b1c665d5ab4b00ab816756f63272dff9802ae72f","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0ebf09d91f98a9a7cd0d6fe8e5f3638052289247e86f3cbe14bdc0d0095b83df","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":17.745182,"date_is_proxy":true,"latency_seconds":4971.341,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.067},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.2-2025-12-11"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"2bdc74ffa93904d36b77acca21389b684c5217027b988ff35985a4a35db8764a","result":{"confidence_high":63.43032,"confidence_low":43.56768,"sample_count":null,"score":53.499,"score_display":"53.5","source_stated_rank":48},"run_fingerprint":"8e1aae6809079f4a99addc8030498d05937fae2234c87846c9d68765fa10c8a8","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ded00d4f682900552f93a2440ca69e2ac23171565ba305c371192d052aeb712f","metric_details":{"license":"Proprietary","variance":131.37041897495558},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.2","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1493c432fab7967c0417626c5ffd83d458fdba1e0fce671f6460def16d6a132c","result":{"confidence_high":1437.9438797539024,"confidence_low":1393.014866644348,"sample_count":1082,"score":1415.4793731991253,"score_display":"1415","source_stated_rank":80},"run_fingerprint":"3723c3b81d83d4645b3c8ad6a3314b1c0bf1e612254208a990789aaac857bc0b","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6393519d4489cc2ad96abfdc","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"52be728cea446fdceb3481166b6d578746fa5dd04fb72a6d76ce04716a83393c","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_high","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"2f9f70d6ae8ad844663d8659d0c05c378aea26b1ed3d3ec0ffc7ca098545bc0d","result":{"confidence_high":null,"confidence_low":null,"sample_count":1647,"score":1480.0,"score_display":"1480.00","source_stated_rank":57},"run_fingerprint":"43d68232a49137170e5099e69c8729da7e1e22220a3b2e27b11f7006b8f8e733","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"6ca1f8f67566a35a1184ecfdb41da99a2263432823d4223d4c77ad39908ffa6f","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_medium","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"796dd5557fc61810aa29a536c4d5f44e40bb2d3a0ea550418867de17c866555b","result":{"confidence_high":null,"confidence_low":null,"sample_count":1641,"score":1394.0,"score_display":"1394.00","source_stated_rank":86},"run_fingerprint":"28967fe201c17e87b7c63ca5655f27e4cc78999cd6c6e2360cb43a9ff520d512","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"37d6b0667b56f40e938f4dddeb6b1d882a27ea8508c7e0e307105c01c9efad9c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"e6131f4e040873de69d09b4259d57b48377e043fe2ec601b4264880f3a9c5655","result":{"confidence_high":1439.52,"confidence_low":1393.77,"sample_count":1055,"score":1416.64,"score_display":"1416.64","source_stated_rank":79},"run_fingerprint":"154ee0d13265ebd82c58e9781267a9b235ebcb872fa80da67bea695d41512cf4","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4b4e04bd5bb4dbf60bb63b4f","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4a57da064f5ca36f30ea1742e9adf16c604432315fb64fffb494d8a57b98e16c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"f0633360f72acc77df9d01cf9d2953ff8c176ba7ef401ce2b267e43c9b253c92","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.19,"score_display":"72.2","source_stated_rank":30},"run_fingerprint":"77994b23d84156d710c85e35cd9152849f7d9b5715a470ba7f1ee0581cfa3efe","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ff397c1e3dacff6a369b03842fe58319d595de4e292c5137a39592ba9f14d7ac","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"dd835091eb7a69a05f5b4b5d4cc925f3a8a18c13df638b8005c44fd70c7c0c97","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.44,"score_display":"63.4","source_stated_rank":43},"run_fingerprint":"37ed2979adbc3ff3fc86e582020d639f4d59f51d5b55267d55126f85e7cf4cf4","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b88ce1d6601cdc625e23ef7325fe0fddabd2d893d3c68860dbe95c2e1b55dc1c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"1228d97c8dfcc485146fb20bcbdea70249c43bfa31a1d8ef6d294ea3de8c65eb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.6,"score_display":"49.6","source_stated_rank":72},"run_fingerprint":"dad525317c93a9bcea07790f8f680bb4ef1145d44caccc730d7aa3a52a4ff409","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40883fb2b66729f88d4e7077","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b22f62f8e2587730dcd84feac50908115d9ddc4f8f8a289f989d404402c4c6f8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"48dedc8719d4d3de8b7dee9d352196dd124d84555c4e89722d9679322c76e24c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.60000000000001,"score_display":"49.6","source_stated_rank":71},"run_fingerprint":"aac6963af33b695bc8b48eb4507c4890c8e5cbbed584e2a14256c07a344d4e9f","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_ff3aeab8b13f468a67900233","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":285,"at":"2025-12-24","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-24","status":"stale"},"measurement_id":"a4c19417f34c824a232ebaf99757f4bc7bb64e0002fd38c543f34b48d494ce05","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"standard_error_points":2.8000000000000003,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_medium","source_row_date":"2025-12-24","source_scaffold":"Droid","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Droid","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"5f589729ec6cc99043ea75ce36de67952658919b83b63c42579dac404da080ac","result":{"confidence_high":70.388,"confidence_low":59.412000000000006,"sample_count":89,"score":64.9,"score_display":"64.9","source_stated_rank":35},"run_fingerprint":"24cffde033ccf2cc6e312326f843040eefb2a857c00c4ee930cd73a18c938f1b","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":298,"at":"2025-12-11","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-11","status":"stale"},"measurement_id":"15e079c76d6dc81896d7449e70d00df682940360d6e9a62a507660533a8da92e","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"standard_error_points":2.7681817951,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.2","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":"2025-12-11","source_scaffold":"Droid","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Droid","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"28914a2f97a8a3662c41c5b514f111cca091de51d339d6035b94376b47d8957d","result":{"confidence_high":70.369456543096,"confidence_low":59.518183906304,"sample_count":89,"score":64.9438202247,"score_display":"64.9","source_stated_rank":34},"run_fingerprint":"583bc1b1030f0cf9252a3a4b7ec368f601ab7d4f48f887ada228fe5c087d2b94","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e2aceefee965467cc3465b70","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":291,"at":"2025-12-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-18","status":"stale"},"measurement_id":"8cbe5b5bdfd15102ac7606160f6a37c1764e1e3c66ac64a63a105564476d1d81","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"standard_error_points":3.0,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_medium","source_row_date":"2025-12-18","source_scaffold":"Codex CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Codex CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"49446febd125f837d24ee17984ca89220a9b51e9105fb815cfe3699b5b1b5ea0","result":{"confidence_high":68.78,"confidence_low":57.019999999999996,"sample_count":89,"score":62.9,"score_display":"62.9","source_stated_rank":42},"run_fingerprint":"5968ee34d1a3ac37cf32625d6e323390f342da29bdec0eb4a24593f6d8122384","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_9c94798e73be04b03372ecd3","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":298,"at":"2025-12-11","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-11","status":"stale"},"measurement_id":"a8219038e1312566a46134a9f9851770ae4b7603389fcd7dce182644bc93e88c","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"standard_error_points":3.0034647333,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.2","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":"2025-12-11","source_scaffold":"Codex CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Codex CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"f1c454a2482bfe1b36c48fd9b5ab8e987f75e44c289bdd11a73db44eec35d2db","result":{"confidence_high":68.80813919186801,"confidence_low":57.034557437332005,"sample_count":89,"score":62.9213483146,"score_display":"62.9","source_stated_rank":40},"run_fingerprint":"9f7ae65da29d459a31e8107da5a89fe0b1d57275c540ef2cd825e7f3016b63d4","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e2aceefee965467cc3465b70","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":298,"at":"2025-12-11","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-11","status":"stale"},"measurement_id":"2931f4cedb4876205a32eb218ff57c3a0ae4d88ade051098f6d9e576aec1ac37","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.2","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":"2025-12-11","source_scaffold":"Mux","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Mux","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"d48960fc9d8dd306d2c57e858d6190d960c87a41c53700e6cd25807451e76300","result":{"confidence_high":null,"confidence_low":null,"sample_count":89,"score":60.6741573034,"score_display":"60.7","source_stated_rank":48},"run_fingerprint":"3a29c587d6afc4e4578333ddca4e46e8bc5a4fc93a41dfda7051f6de2f72e22f","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e2aceefee965467cc3465b70","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":298,"at":"2025-12-11","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-11","status":"stale"},"measurement_id":"b79e4dfa2140e3cf2106d81210a529719b7130b5192070bd08f30716fa56609d","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"standard_error_points":2.8910174223,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.2","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":"2025-12-11","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"63cdecedb4d5a17a3db3b67fe102a633bd2b3a0a9b8f0004fbe92290c404dc9f","result":{"confidence_high":59.711337967907994,"confidence_low":48.378549672492,"sample_count":89,"score":54.0449438202,"score_display":"54.0","source_stated_rank":63},"run_fingerprint":"4e07dc08ce82af9d2f1d47a27774ca4fe213a5409caa7d64c81b9276b40e1190","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e2aceefee965467cc3465b70","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":153,"at":"2026-05-05","basis":"evaluation_at","evaluated_at":"2026-05-05","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"12d3e28e4df971469926df109329ebacb8c8bb6058d1d9f0d8ae85cde8ffe412","metric_details":{"cost":null,"pass_2":23.88,"pass_3":20.62,"pass_4":18.56},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"gpt-5-2_sierra_2026-02-26","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"fddc1da1ea5e3ec0812c175281f75e6b59702134f5a28ebade6fbbc00b5ce9fe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.22,"score_display":"32.2","source_stated_rank":null},"run_fingerprint":"13de9ec5318531b13c1d9d0495b0614cb9fbf884960e2f4a960730bf2ba25e39","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":132,"at":"2026-05-26","basis":"evaluation_at","evaluated_at":"2026-05-26","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"d5adf736cdff1ad159c4d28a5765a17018ca319d08df8bed2c9b86651c3542e1","metric_details":{"cost":0.2433424974226804,"pass_2":7.22,"pass_3":5.15,"pass_4":4.12},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"qwen_embeddings","submission_slug":"gpt-5-2-none_sierra_2026-02-26","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"310f961acb7ebfcb152b2c29865dbe407576c8ab0f9a9a8504c97608ccd65b0b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":12.63,"score_display":"12.6","source_stated_rank":null},"run_fingerprint":"95ffc2451fcea2e79ecef9c72839ddc48c95b2e534fabbe12985ffc15d0e60ef","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_ff3aeab8b13f468a67900233","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"4426c94fd8fb35d6d48b96094b86ab3ad420fc746aa7465c1ac352235c0431b9","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 87 scored items with a 95 % Wilson interval.","expected_trials_per_mode":261,"normalized_gain":null,"recomputed_date":null,"selected_trials":261,"skill_lift":22.0068},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"codex","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"codex","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"de4ee8be7ebbb4075f4b810c6a6dcd2e1f701030bba81db0d7f2320352479107","result":{"confidence_high":61.908934,"confidence_low":41.356094,"sample_count":87,"score":51.7046,"score_display":"51.7","source_stated_rank":12},"run_fingerprint":"e8f300a63c2e28ffb5ad2d802aec38f172b7a87b821113d7da1cdbf026f51851","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5abeb85214411e952b3bbd17","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"5154c96488cc8b20280e5ec048a3c8d6db761fe15a9275f74623932c271b8aaf","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 87 scored items with a 95 % Wilson interval.","expected_trials_per_mode":261,"normalized_gain":null,"recomputed_date":null,"selected_trials":261,"skill_lift":22.0068},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"codex","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"codex","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"fb812031761af2e4bf7d902ad4e16eab19781fbf07e90c9eda8d41c09aed7621","result":{"confidence_high":39.991876,"confidence_low":21.120844,"sample_count":87,"score":29.6978,"score_display":"29.7","source_stated_rank":15},"run_fingerprint":"63caf965b4f3861413c3051fd91f9402da5ae842f2ebe70eb961aa956a1f0a07","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5abeb85214411e952b3bbd17","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"d8fa14ca37dcb22eefb124916629c6a18111979d883a48070cd26f9a6ec5073c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.484","mean_standard_error":"3.5","model_release_date":"2025-12-11","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.8,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5.2 (xHigh)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"d698e27cf225090fbf958e52f466401d39ea3fd2b26239b9f09de07f48dc3646","result":{"confidence_high":41.848,"confidence_low":26.951999999999998,"sample_count":480,"score":34.4,"score_display":"34.4","source_stated_rank":19},"run_fingerprint":"7160a4d9d75d424c7c8e3f2a5ad7a703c12fc1dd25c46687d0aff79153f16f41","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_4a86c49777c47cb202b9dfc5","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"33c56c730b4bd247775dde8d25d5416371244daf788a6b9a7966bf17349ca2f5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.387","mean_standard_error":"3.2","model_release_date":"2025-12-11","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.2,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5.2 (High)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"0fc6950207ad9e1c5273e79a17fbf69151eb89dde120c62b8e99dad3a2be6f5f","result":{"confidence_high":29.272,"confidence_low":16.728,"sample_count":480,"score":23.0,"score_display":"23.0","source_stated_rank":33},"run_fingerprint":"533e40b1566db8eeabbbdf4896018e7a8183dea46c39d8695f1f3c2f21266b72","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"896e154def8f6cb1cce7af43386eff58c1cb49b8ecb1ff455c5e64722ba43c98","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.387","mean_standard_error":"3.2","model_release_date":"2025-12-11","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.2,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2 (High)","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"49b79930db3c98a09972d3e4b05eda108d42446605b9dfd27d0dda55353afd55","result":{"confidence_high":29.272,"confidence_low":16.728,"sample_count":480,"score":23.0,"score_display":"23.0","source_stated_rank":33},"run_fingerprint":"41df232adbccc4cb95cbe2b9d83299f2808250ee4fe0cd7bc5bbde30f4919ac1","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_da6fffb3400932260676d0d4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:03:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:03:18Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cd11e4923ff70e1c795e4bbd90bfac5836b96bc3614a13bad7820f0e2008e9af","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":2.23,"leaderboard_last_updated_at":"2026-04-12","license":"Proprietary","model_link":"https://openai.com/zh-Hans-CN/index/introducing-gpt-5-2/","total_cost_usd":85.65},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"FC"},"run_count":null,"scaffold":"FC","tools_allowed":null},"protocol_fingerprint":"6a9c99bccbca11748bdf70a2bdd6eb864726c94bf4a029f84ab95898fe96d192","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.87,"score_display":"55.87","source_stated_rank":16},"run_fingerprint":"4be67636ef17c1b568b6dfa8779ab07fb926850d78ce4585cab0018370b48d3e","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c45390ebff98a31c8653458d","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":38,"at":"2026-08-27T22:30:42Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:42Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"95e817dd13d98ce45fa263e840b6c7ba61ef9ef4cd0d383486b0c86a52f59e37","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":4.21,"leaderboard_last_updated_at":"2026-04-12","license":"Proprietary","model_link":"https://openai.com/zh-Hans-CN/index/introducing-gpt-5-2/","total_cost_usd":164.58},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prompted-function-calling","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"Prompt"},"run_count":null,"scaffold":"Prompt","tools_allowed":null},"protocol_fingerprint":"564fbea687964bbbeb4c2a4d92f977e2c00894a357d9ecf0f2b928b7737d37ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.27,"score_display":"45.27","source_stated_rank":38},"run_fingerprint":"0a490dcf94d4d42498334c16785e43d75970141e975490ea17f20df139e675b8","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c45390ebff98a31c8653458d","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prompted-function-calling","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"19a7ed6a006bf53b743b42557f789cd4f04956404b6035c4b34718c0e97b0ca8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"6da35d09238c9df45d2f323fe5aeb267713f7fa3fa3027c58f086ef6272f3446","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3591.3259999999996,"score_display":"3591.33","source_stated_rank":42},"run_fingerprint":"3530d6ab6e571768a9d4ae2272e46b9fefee0c88d7830f9c582cf10e18855903","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_71d5b9dc73fff7d1c5cd8972","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval","metric":"win_rate","name":"GDPval","release_id":"release_730527fa346470703bd19e21","split_or_window":"expert-comparison-win-rate","unit":"percent","version":"GDPval official leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:05Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:05Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0631cc5b56f575f0abe8594beb6362b382c3289a354eba8e1eedd8b59b49f6e1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-11","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-2","name":"GPT 5.2","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.2","source_effort":null,"source_model_version":"gpt-5.2-2025-12-11_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenAI GDPval expert-comparison protocol","tools_allowed":"as reported by the GDPval leaderboard"},"protocol_fingerprint":"5a3fcb25a7d145e0ee5aac9c23e0f8e4c772626f45806ef93461b4abdb803807","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.7,"score_display":"49.7","source_stated_rank":1},"run_fingerprint":"aade77e54817691a60250e4f76d69908b7be3389bf016d595868269f22a42bd1","source":{"content_hash":"d4eac25e0863d96ee45cc545fc896e5667d7022da8b3bc3902533ed87fa8b836","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gdpval","id":"epoch-gdpval","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GDPval","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gdpval"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_ff3aeab8b13f468a67900233","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1739e807d063980e495232af5edfd50211b0ea60c8ce4d6274fddfcee9cb70d9","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-11-18"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 3 Pro; model release: 2025-11-18","source_accessibility":"API access","source_model_name":"Gemini 3 Pro","source_model_release_date":"2025-11-18","source_model_versions":["gemini-3-pro-preview","gemini-3-pro-preview_low"],"source_reasoning_efforts":["low"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3deb2eee2c455dd742ef2f3024dd6bd89c98da9eca09a6d31a9f0a95d0777edc","result":{"confidence_high":155.17,"confidence_low":150.6,"sample_count":null,"score":152.95,"score_display":"152.95","source_stated_rank":39},"run_fingerprint":"2ec98df75fa567b46dfda72dc09d755a8c96a0d29384346d7ac7f33c90303bff","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_21273f06524672cf31a6a06e","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"962f9e90d8680f0eb14c900465f069dce51dce473f2a38424a38d76a4723aa5c","metric_details":{"license":"Proprietary","variance":3.7535577942103},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1489.259521100251,"confidence_low":1481.6650131642346,"sample_count":41910,"score":1485.462267132243,"score_display":"1485","source_stated_rank":19},"run_fingerprint":"13bfdd7ed40ff6a690614140a0806bbe17820a16b159ce86f4684d2910c59afc","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0271df1781497779ba7d4b50","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":319,"at":"2025-11-19T23:50:49Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-19T23:50:49Z","status":"stale"},"measurement_id":"2e67417a0e3f4f9d4a3dca765da6354e3ca94640ecef4fd23f183d921574aebc","metric_details":{"calibration_error":57.0,"confidence_interval_half_width":1.9,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=gemini-3-pro-preview; createdAt=2025-11-19T23:50:49Z","source_row_created_at":"2025-11-19T23:50:49Z"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"gemini-3-pro-preview","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"115793457a711fb092acbd18cda3eb4fe740d66a2e0492d281f3d202125c938e","result":{"confidence_high":39.42,"confidence_low":35.620000000000005,"sample_count":2500,"score":37.52,"score_display":"37.52","source_stated_rank":5},"run_fingerprint":"16e5061edad57a290a27c190df42f802c9ceffaa8099f6fdae0825dba7ea1d57","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=gemini-3-pro-preview; createdAt=2025-11-19T23:50:49Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b228b8ba7c4906d4189d84aa","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":321,"at":"2025-11-18","basis":"evaluation_at","evaluated_at":"2025-11-18","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4ca1b58eb405f73a32c27734074a179c5a8de26382771b1c70f65c2fc0665742","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/google-gemini-3-pro","source_model":"Gemini 3 Pro"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d6c8a5d511cf0e665c2450e02c0f8df8ebef20aeede3e78c2fd07a8ba6376a88","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":38.3,"score_display":"38.3","source_stated_rank":15},"run_fingerprint":"1f2f3cfc16f9a8f3420f47df48da10db56870131be097f06760cfb76f3320173","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_446a7d9930d2d0d768a97786","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":326,"at":"2025-11-13T00:00:00Z","basis":"evaluation_at","evaluated_at":"2025-11-13T00:00:00Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ef2c752a0f5f939c85224c07757e6e556cdcd6632c0b5358c5048c3096d87ca6","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3-pro-preview","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b3f27ff774166a33798e6609b791b03ff7b9c2a1ba3229d991c2865b8aa97513","result":{"confidence_high":74.996110833,"confidence_low":69.209897747,"sample_count":null,"score":72.10300429,"score_display":"72.1","source_stated_rank":5},"run_fingerprint":"9e3112cb000d245f47afe7014158f27eb1b8da98e48076c2f3603577b6291d01","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_99f15e2ba246f14793e1e251","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d504b9d757898245e0bc5ef3624a504a87ea247d2caf4e6ac8b392ee1c8bed3e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.089573,"date_is_proxy":true,"latency_seconds":61.34,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.389},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":90.909,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65535,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-pro-preview","zero_shot_accuracy":92.424},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"ebd4e59abc04d287b28b24df8427bf6279150f95c91ad989a360dbe09b42a689","result":{"confidence_high":94.38844,"confidence_low":88.94355999999999,"sample_count":396,"score":91.666,"score_display":"91.67","source_stated_rank":18},"run_fingerprint":"d7cc787e86f199e3a86b941886b646949e778bef3f6de23ea52c1b1303f95e0a","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_adb9d064fb6ec60affbc6752","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":319,"at":"2025-11-19T15:16:10.279Z","basis":"evaluation_started_at","evaluated_at":"2025-11-19T15:16:10.279Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"db7ad5e0fe39daecea57f2e7fe2e35059159604fa7b2267860ebb18e360edcd3","metric_details":{"best_score_across_scorers":"0.9261363636363636","model_release_date":"2025-11-18","stderr":1.6526963574741},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Cg54udQGcTCnfDGX2qePLZ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCg54udQGcTCnfDGX2qePLZ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Cg54udQGcTCnfDGX2qePLZ.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e195af6942ef9894637ae6af5c2cb86ad053394e02c7404acd15fa541eade6cd","result":{"confidence_high":95.8529212242856,"confidence_low":89.37435150298712,"sample_count":null,"score":92.61363636363636,"score_display":"92.6","source_stated_rank":25},"run_fingerprint":"ef8c0658188f29517b3810fef3c49ae8aed9f79a3b9a0978b829d4b8d2c20b8c","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cd6e7ab8d3edb3128b842eaa","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":326,"at":"2025-11-13T00:00:00Z","basis":"evaluation_at","evaluated_at":"2025-11-13T00:00:00Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8f93a5bf5b37c37e5c93b127face8a44879c2518ce61bf854663218602eb9244","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3-pro-preview","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1293cd1bf451b129593fac74aec11a4f167f33db061befa395ec8102442e3ef0","result":{"confidence_high":96.884254069,"confidence_low":89.984432791,"sample_count":null,"score":93.43434343,"score_display":"93.4","source_stated_rank":3},"run_fingerprint":"a65f5cd80ea4c183278d1f62fed21956cf6beb3994467b9f6f85b102582f29bc","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_748a5b98418561686207c96d","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d29d42a59109972ce45f593c1d8213cce98f7321ea4164f28f97c6b35f3b1d40","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.036926,"date_is_proxy":true,"latency_seconds":24.272,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.295},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65535,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-pro-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"72046cfa678d1b5f298a84b6e9297c90f65659b20d15b19b0a799e730a380873","result":{"confidence_high":90.6802,"confidence_low":89.52380000000001,"sample_count":null,"score":90.102,"score_display":"90.1","source_stated_rank":7},"run_fingerprint":"8eb3d6ddde3095c1d1b9c6fd2d1499adc76df42dee4c0ec3e4a4c95859015505","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7c54b62064912898a8a23a9a","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":319,"at":"2025-11-19T15:16:13.231Z","basis":"evaluation_started_at","evaluated_at":"2025-11-19T15:16:13.231Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"6403b0fe706c62d5c0b346bbe9b825f70a3591b5dd61f4709705d15bbb57dfe6","metric_details":{"best_score_across_scorers":"0.9138888888888889","model_release_date":"2025-11-18","stderr":3.658872186755414},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4QM4h7ndQiDRL2anjKeddK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4QM4h7ndQiDRL2anjKeddK.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4QM4h7ndQiDRL2anjKeddK.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9dbd0a5981231b06590c4da0465a446a5729320593618e11466b8ee4c9e19f03","result":{"confidence_high":98.5602783749295,"confidence_low":84.21749940284828,"sample_count":45,"score":91.38888888888889,"score_display":"91.4","source_stated_rank":63},"run_fingerprint":"a69d514298daf96ec934591f7e0fa85e693fffa59bd522fcda7465c4407039bd","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2f83be29a9f1348d44b039f8","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5f49235eeb1815465b7be548342d6ccc0cb047cbf88b357a73b3120dab6dbc89","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.8109,"model_release_date":"2025-11-18T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-pro","model_type":"CoT","upstream_model_id":"gemini-3-pro-preview"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"99a1d91d545a77cd7589ab38d5e6c23ee720a21c66735bb8e5f3e86f432a91de","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.11,"score_display":"31","source_stated_rank":117},"run_fingerprint":"8cdc55d6bea532a666feb172dddc6cf3487b1b1ee14bee8669e4fd88ac36ca3e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_55859642c21787720a9cc539","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8f75ac1edba8213dd9583922d7664e1e447c898996eaf84a1c2cc8d0563385f5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"97162363d7b2423d2e3227f8b74b98edf2e197945159155bc246a5107cadd20f","result":{"confidence_high":36.070572,"confidence_low":26.548326,"sample_count":360,"score":31.11,"score_display":"31.1","source_stated_rank":117},"run_fingerprint":"26783330286048c97b83ed62a428c138c444fc08d46b5c39c609eb9f6a7544a4","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_eeb8077bcff80db200310c27","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"55f124871028738e1382e73b2b8cd3cf907cfb634c80a357579b108257e24c24","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.010159,"date_is_proxy":true,"latency_seconds":48.102,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.795},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65535,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-pro-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e932406a85adacb440e952ebcf597dad5543c0bb49a521f992e5b26d3df7f019","result":{"confidence_high":89.0722,"confidence_low":85.9558,"sample_count":null,"score":87.514,"score_display":"87.5","source_stated_rank":14},"run_fingerprint":"eb431311996fa2498900d365faa45e9a7a4511cfef648b4198c0531002d30374","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_adb9d064fb6ec60affbc6752","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dc677026f30c7e15c5b284e384d15ee84391d260cbb2d0c79fa605f52a666564","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","cost":"95.3","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini-3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"9162a8226815c167a678857e1353c87c3a641643f4952f0fa1f5b1558dcb125f","result":{"confidence_high":15.266013,"confidence_low":2.958611,"sample_count":71,"score":6.9,"score_display":"6.9","source_stated_rank":105},"run_fingerprint":"12eac4297237c1bd3567396c1b5a63c4b3b7d8d175db2582fc0138fe57a944fe","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_06cf7a8f0e7a9f715887e058","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4cf081fb301e98cfba09a5a6c3f239c5c0fb32b027a6688523c88c068fb303e0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"5598f250eee0f48592d2053da9bdea88c7a82ed2e827eebbbc2bc6954db727a2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.4,"score_display":"76.4","source_stated_rank":7},"run_fingerprint":"09cf687ae9db68ae3004df1664fe6e1315330a79bb9fe2d7e0f9c9155c7d28df","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2f936e9456cd58720653f4cf","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":300,"at":"2025-12-08T14:58:39.203Z","basis":"evaluation_started_at","evaluated_at":"2025-12-08T14:58:39.203Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"465d089480bbf0bd30cdf4aa56d894137aafeaca10cdd4e68f5ef674669f4572","metric_details":{"best_score_across_scorers":"0.31","model_release_date":"2025-11-18","stderr":4.648231987117317},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Ey65gb5LG3RdtZxMweDfAU","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEy65gb5LG3RdtZxMweDfAU.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Ey65gb5LG3RdtZxMweDfAU.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"922ce9dd79107bc61b8f38081ab1e6394e6920e33c9b977af001b5ceb9add71d","result":{"confidence_high":40.110534694749944,"confidence_low":21.88946530525006,"sample_count":100,"score":31.0,"score_display":"31.0","source_stated_rank":48},"run_fingerprint":"39a0688d188634bbcb098b0db1c1c46dcd55285f19f5726411120456552f72e6","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d4b574d49820900891724f4b","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b5c55628a9f47898f245aeab539b5c8f25ec03fb962f68634021e73061426b5b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.4926","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a61cc4282fc3788ccaa14ffbf2fe051a812472f438d814c2cfb9795c40923e80","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.0,"score_display":"75.0","source_stated_rank":125},"run_fingerprint":"09a43b5cffb5473d6f76d356b456c8462ff5004df3a69445bce3b9f4da8c766f","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_eea65165256ff8318729c400","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ccfa04edd67a9336051b6deb00e6b77826bcf820de1151f1ca3579cceab877d7","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.771547,"date_is_proxy":true,"latency_seconds":394.488,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.901},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-pro-preview"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"c042c892e3ccd68526441b4ce338099c325acfda87c99c30454bf916d975c3c1","result":{"confidence_high":80.12596,"confidence_low":72.67404,"sample_count":500,"score":76.4,"score_display":"76.4","source_stated_rank":37},"run_fingerprint":"d977d10bbd02e42c50f5f654f61867df1a2bc9c169495e819910e97eb6aab6ca","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_adb9d064fb6ec60affbc6752","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":221,"at":"2026-02-26","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-26","status":"stale"},"measurement_id":"624ecf4a941c67440f47308e2ac0935b01921bba1678cdd6afea9c1e32ef7859","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2026-02-26","folder":"20260226_mini-v2.0.0_gemini-3-pro-high","logs":"s3://swe-bench-submissions/bash-only/20260226_mini-v2.0.0_gemini-3-pro-high/logs","model_display":"Gemini 3 Pro","model_org":"Google DeepMind","reasoning_effort":"high","resolved":69.6,"site":"https://ai.google.dev/gemini-api/docs/models","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: gemini-3-pro-preview","Org: Google DeepMind","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260226_mini-v2.0.0_gemini-3-pro-high/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"7a2a684013cd145b255c77171c1528b749f4dbf250e53bd201fb07d67aabd44e","result":{"confidence_high":73.469862,"confidence_low":65.431253,"sample_count":500,"score":69.6,"score_display":"69.6","source_stated_rank":32},"run_fingerprint":"f676fb278eb4da6a2460bc9a5cd8347ac5690d7665eecfefdaf51c6da1902b5c","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_adb9d064fb6ec60affbc6752","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":321,"at":"2025-11-18","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-18","status":"stale"},"measurement_id":"456bfec0a2613d368dd1b6a7af9bca919958de1eb83007206122d8110bf65e64","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":true,"date":"2025-11-18","folder":"20251118_mini-v1.15.0_gemini-3-pro-preview-20251118","logs":"s3://swe-bench-submissions/bash-only/20251118_mini-v1.15.0_gemini-3-pro-preview-20251118/logs","model_display":"Gemini 3 Pro Preview","model_org":"Google DeepMind","reasoning_effort":null,"resolved":74.2,"site":"https://ai.google.dev/gemini-api/docs/models","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: gemini-3-pro-preview","Org: Google DeepMind","System: Attempts - 1","Mini: 1.15.0"],"trajs":"s3://swe-bench-submissions/bash-only/20251118_mini-v1.15.0_gemini-3-pro-preview-20251118/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"c521fe9e15c03986d841965ec0541ac8c44b48dac7edbccd98bfb2371bad4b6e","result":{"confidence_high":77.840443,"confidence_low":70.190525,"sample_count":500,"score":74.2,"score_display":"74.2","source_stated_rank":15},"run_fingerprint":"a692d197f1b2c94d2b8f9d074859f37c4ffe3660abd4ddfea25b89e151100d39","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ee842faf8f9133428c458e0a","provider_config":{"source_id":"official-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":234,"at":"2026-02-13T10:43:27.233Z","basis":"evaluation_started_at","evaluated_at":"2026-02-13T10:43:27.233Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"6d0405f279917ec702bd6e7b1247b828a4efbe9df411b9d84250ee704c147180","metric_details":{"best_score_across_scorers":"0.7293388429752066","model_release_date":"2025-11-18","stderr":2.021641171153417},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"5XdYZBG5Wy8DUMRqcYv4fn","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5XdYZBG5Wy8DUMRqcYv4fn.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/5XdYZBG5Wy8DUMRqcYv4fn.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"4d73b3cc36216f9d7f5a5e49a01b3a41aafb055a4a7a460905228006acff17bd","result":{"confidence_high":76.89630099298137,"confidence_low":68.97146760205996,"sample_count":484,"score":72.93388429752066,"score_display":"72.9","source_stated_rank":22},"run_fingerprint":"047ed4de87d92bc9df04bb3ab53cc6716613ce0938ee842ede6842b262b03f12","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4483466626b3431da25c31d7","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":319,"at":"2025-11-20","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-20","status":"stale"},"measurement_id":"f2df67894b553caf3e1a509327175287020dad4c9bff7cb9658824a4b3745962","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"live-SWE-agent","checked":"false (See README.md for info on how to get your results verified)","date":"2025-11-20","folder":"20251120_livesweagent_gemini-3-pro-preview","logs":"s3://swe-bench-submissions/verified/20251120_livesweagent_gemini-3-pro-preview/logs","model_display":"Gemini 3 Pro Preview","model_org":"Google DeepMind","reasoning_effort":null,"resolved":77.4,"site":"https://github.com/OpenAutoCoder/live-swe-agent","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: gemini-3-pro-preview","Org: UIUC","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20251120_livesweagent_gemini-3-pro-preview/trajs"},"run_count":null,"scaffold":"live-SWE-agent","tools_allowed":null},"protocol_fingerprint":"11bcd39ed7320d3431044fac9ca8316256a38a87e5e1ca79a8a891b8b45eccd8","result":{"confidence_high":80.849081,"confidence_low":73.53309,"sample_count":500,"score":77.4,"score_display":"77.4","source_stated_rank":4},"run_fingerprint":"1a8d4c6c954dfd701ad3e1b56b659761a00ec3040402bb0cb052ce17d0978687","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ee842faf8f9133428c458e0a","provider_config":{"source_id":"official-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_1c18f4398e32b2e0fe7f7d86","split_or_window":"2025-12-01/2026-02-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":246,"at":"2026-02-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"08ad310b01ca4ef1f0154b5b5c62604db450fd74b5d33b118cb07bbcddfe6fd6","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-11-19","pass_at_5":63.541666666666664,"potential_contamination":false,"sem":1.9951573123017194},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"Gemini 3 Pro Preview__tools","window_from":"2025-12-01","window_status":"historical-post-release","window_to":"2026-02-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"aabe485e331299acc0f7029513ee3a87f1b24a21da19427abfac925db891570b","result":{"confidence_high":56.65392938474296,"confidence_low":48.83291272052022,"sample_count":null,"score":52.74342105263159,"score_display":"52.74","source_stated_rank":null},"run_fingerprint":"f4e92d07c2e51715b7e2c507a4278775841d0f1a1277c63bbe9e1e32eb28c8ab","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2123e902a08dee70792ebf5f","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":312,"at":"2025-11-26T20:56:30Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-26T20:56:30Z","status":"stale"},"measurement_id":"9dd794110eaaf31406a903313c9809f95cc13b24c44195e49faba37cfa0884f6","metric_details":{"confidence_interval_half_width":3.6,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gemini-3-pro-preview; createdAt=2025-11-26T20:56:30Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2025-11-26T20:56:30Z"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-undisclosed-harness","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":false,"protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"gemini-3-pro-preview","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576"},"run_count":null,"scaffold":"submission-specific","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"8784a8d3bd980e3e84c1f88104cc5f1525f1c71dfed8e53d28fe4cf7adb65be5","result":{"confidence_high":46.9,"confidence_low":39.699999999999996,"sample_count":731,"score":43.3,"score_display":"43.3","source_stated_rank":5},"run_fingerprint":"4e350ba07104f56694e63a94c3f1209fc4d43171111767f336a23eacec6e98fe","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=gemini-3-pro-preview; createdAt=2025-11-26T20:56:30Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_66660e2414c0f7c659548ae2","provider_config":{"source_id":"scale-swe-bench-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-undisclosed-harness","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_ae9931700b0a66fc6502e81e","split_or_window":"google-single-attempt-comparison","unit":"percent","version":"SWE-bench Pro Public · Google single-attempt run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-19","status":"stale"},"measurement_id":"d7108a4aba61037938f760a1b818a4e439ef9f042dc067327e083e843d062a1d","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","evidence_sha256":"4c75baf400ffa3d11b2095ddd52c7fac9689fe3a545396e8f46d0fb0297ebd20","kind":"reviewed-static-report","published_at":"2026-02-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","version_id":"8280865cd0e38d4df4d25ebd0ff7d528ea0e07796dcba3ea1dea08b2faff6f9a"},"source_locator":"Model-card results table · SWE-Bench Pro (Public) · Gemini 3 Pro Thinking (High) column"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"thinking high; single attempt; exact scaffold, task revision and sampling settings not disclosed","reasoning_effort":"high","result_published_at":"2026-02-19","results_as_of":"2026-02","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"026e6cfdaf6ed476df3c435c09dabcd9e63ecfc1d0dd8c121f3fac7f581b0121","source_published_at":"2026-02-19","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Google comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"3ca38e7af73a438235d26a51fa910b0280e4e1b599f4a038fc13e8e748479cd5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.3,"score_display":"43.3","source_stated_rank":null},"run_fingerprint":"71e1cda843ce0fdfd31c914106a5833ccbcbe260f21239b00bc2abb11ead8651","source":{"content_hash":"e6b36e17e4cfb89dc185360e76dc7116af66b9615badb20bc3b4ba6fd36baeb5","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","id":"google-gemini-3-1-pro-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card results table · SWE-Bench Pro (Public) · Gemini 3 Pro Thinking (High) column","name":"Google DeepMind · Gemini 3.1 Pro Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-1-pro/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_adb9d064fb6ec60affbc6752","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":311,"at":"2025-11-28","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-28","status":"stale"},"measurement_id":"06122f5c51ca7eaba251e606a0eec070c36ac119aca2a848c90b051dd71a1cc7","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2025-11-28","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"93d2c24bbb66465156fda6199944da881b585d0d0009827798ac52c8d19539c6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.63,"score_display":"18.6","source_stated_rank":15},"run_fingerprint":"c62ab866596b3e67639b785bcbcaa10c24887abe1f5a8f68f6dc0d88bdef57f8","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f7289fba54bab2a09b2c9891","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c7269ead7087667c3da4c30bfcc40d0ecb9ecd843f7332fb65322117e7af3b3f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.12729,"date_is_proxy":true,"latency_seconds":87.226,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.982},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":null,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"google/gemini-3-pro-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"9736a3ce929e0f8ba22537444715057806f8c2843f25d2ab4fb56257e0a8c7b6","result":{"confidence_high":88.33171999999999,"confidence_low":84.48228,"sample_count":null,"score":86.407,"score_display":"86.4","source_stated_rank":23},"run_fingerprint":"7f2ca89f218a4c211cf72ff39b90ae6132a4ed2fd894220bf7d84fdc5038a335","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fef4a508fae5caf5110e0767","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"780f8036219a6850d524e3dc02e7b0137ae3821e7a9717be9594ae8e331be98f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.259385","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-pro-preview-high","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"b90b91119d6e1c0d23d68ebf852ba81fd02dcd362d1175a124eb4f432f65cf7b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1176.75,"score_display":"1176.75","source_stated_rank":38},"run_fingerprint":"d2e46b8be574a754ee98078a544cab0629b0264efca707baa4c57c3c2c8064c3","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_adb9d064fb6ec60affbc6752","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bebe1d11ed3f5b4172d685ad986451355651a6d6993da94921e185f291deb4d9","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":6.417592,"date_is_proxy":true,"latency_seconds":10398.545,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.061},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-pro-preview"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"ef2f9524482c65626283b0af2866832dce91df38f466be1f619cbfeded025df6","result":{"confidence_high":20.29956,"confidence_low":8.300440000000002,"sample_count":null,"score":14.3,"score_display":"14.3","source_stated_rank":84},"run_fingerprint":"b637b598ddc7ea00d4c8997238da0c44feee98cd1097438cd947e57ddcfea928","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_adb9d064fb6ec60affbc6752","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"efe3c8bca2f25718153e01d534f5c7073a2f135be38c9d4f51e424ef68adca74","metric_details":{"license":"Proprietary","variance":18.39388245348863},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3-pro","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"74fce235d0a7d0dd84ccccc20c294b909a8632f17404399d5c99747908a59388","result":{"confidence_high":1447.3938068313628,"confidence_low":1430.5819845627607,"sample_count":14092,"score":1438.9878956970617,"score_display":"1439","source_stated_rank":71},"run_fingerprint":"2f2d01748b0ce4da28a968ed032c2649850aa1e26b5fc49afb3ddb34844c550c","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8b1d5607f64765be6892d3c5","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"c283417c4f862d815ea5420125e94177690bf066dd65fd7af0611003f1090d04","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"ab4fec85d9772804c30c76844c06403dc58223874bd28d13bd67bfb0f6171bcb","result":{"confidence_high":1447.32,"confidence_low":1430.22,"sample_count":13573,"score":1438.77,"score_display":"1438.77","source_stated_rank":71},"run_fingerprint":"f0e4cdae6dd1bd6c1bf91062a512efdc81a928abc559215f8befe2031006c8ef","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_56c2d7397579ae26e30cef08","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ff711d76eaced0afe59c2bcfe1dc6443ea16b9071e6d8f387ff2611ba665a24c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"823d1839ab557ec28302622ebf9b212fbee470eb44ec550284a1997c2dbb10db","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.93,"score_display":"69.9","source_stated_rank":34},"run_fingerprint":"1ef0e9919248aa598fa89ab142421a94c373201b0d7682e55db532dc194fbb41","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cd9c62d22639dee9b3c647d7","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":242,"at":"2026-02-05","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-05","status":"stale"},"measurement_id":"c797f8e6d59379f915721dde604ee73e1bb1f4c3e8175da6fe3081687209f664","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"standard_error_points":2.6,"upstream_source_url":""},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2026-02-05","source_scaffold":"CodeBrain-1","upstream_leaderboard_url":null},"run_count":null,"scaffold":"CodeBrain-1","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"cb40f750002271f59384a17ae5681e4d1c41a051f4833b1fc927ae2fd609b829","result":{"confidence_high":67.296,"confidence_low":57.104,"sample_count":89,"score":62.2,"score_display":"62.2","source_stated_rank":44},"run_fingerprint":"f59ceb9978a130dfff61e24e02ee005725845d3c3d1737f7f03ab6bc050c0f46","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f06c5117dc5d721d9cb72b2a","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":321,"at":"2025-11-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-18","status":"stale"},"measurement_id":"4487384662b650558819b29e1df8ff6487a1bfe6bf6e89f4937d93446bf9fcc5","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"standard_error_points":2.0886618713,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2025-11-18","source_scaffold":"Ante","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Ante","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"1154030cf2734171951e7f006ae0706a6476c2329962f915bfbaf64e6c9e60f3","result":{"confidence_high":73.519495120448,"confidence_low":65.33194058495201,"sample_count":89,"score":69.4257178527,"score_display":"69.4","source_stated_rank":27},"run_fingerprint":"f43880a725ce0b6b8ff3bb13e8ab45a77e1c71fa3bb2b156d6c77f398054db35","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f06c5117dc5d721d9cb72b2a","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":321,"at":"2025-11-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-18","status":"stale"},"measurement_id":"0be98c683d49c387d96f9a2609db97a2b7ce507468fd25da313e2caa8718347f","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"standard_error_points":2.0658909863,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2025-11-18","source_scaffold":"SageAgent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"SageAgent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"6920418076a49c6f2724f54d3bf445b282856ef355ca42f345ece769b574ce7e","result":{"confidence_high":69.217685658948,"confidence_low":61.119392992651996,"sample_count":89,"score":65.1685393258,"score_display":"65.2","source_stated_rank":33},"run_fingerprint":"868a0e17c1baca5bce52a6fcae16962e30819a4b49328fce5e71d85d44053ba8","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f06c5117dc5d721d9cb72b2a","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":321,"at":"2025-11-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-18","status":"stale"},"measurement_id":"559f4898f9ee18788ad93a58881f87605476ac2c1692c916b2a6837cb2cf05ed","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"standard_error_points":2.6057340168,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2025-11-18","source_scaffold":"CodeBrain-1.5","upstream_leaderboard_url":null},"run_count":null,"scaffold":"CodeBrain-1.5","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"7dec51937cdbc270605ef1c7668ba9f8d3d475be25cc11a6b7730dea2a84d75f","result":{"confidence_high":67.35442968412801,"confidence_low":57.139952338272,"sample_count":89,"score":62.2471910112,"score_display":"62.2","source_stated_rank":43},"run_fingerprint":"fe27e08612efb999465804f9ee4747c031698fab85d9265c352e083e5b9bbea4","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f06c5117dc5d721d9cb72b2a","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":321,"at":"2025-11-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-18","status":"stale"},"measurement_id":"36d158b1a7e6cfbac6f8cef28fd8059c111a76c465e3c1e383d9b4305adfc50e","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"standard_error_points":2.8202524731,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2025-11-18","source_scaffold":"II-Agent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"II-Agent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"68c04c07f9b12f25ac32bf9699d9e75020dc639024b42dbf307579c51f199e20","result":{"confidence_high":67.32544765627601,"confidence_low":56.27005796172401,"sample_count":89,"score":61.79775280900001,"score_display":"61.8","source_stated_rank":45},"run_fingerprint":"b9c2525e64d808def1e211d0176dd52cdabccd5e39ae0ebde5a25f9f3db02403","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f06c5117dc5d721d9cb72b2a","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":321,"at":"2025-11-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-18","status":"stale"},"measurement_id":"22f9d4720c05578b2b2e29cb6c9b70d73b4934a65a8003329ed0ac824c531292","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"standard_error_points":2.7856468377,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2025-11-18","source_scaffold":"Droid","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Droid","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"b4bd3725be9774d12a09f812aa04d96ae54c47786c510cc19992dcc23547e7db","result":{"confidence_high":66.583463307492,"confidence_low":55.663727703708,"sample_count":89,"score":61.1235955056,"score_display":"61.1","source_stated_rank":47},"run_fingerprint":"6ab79d532f2bb7266a549a7fe7be82718af4048a2f3c1edc2f927ca946ea4b11","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f06c5117dc5d721d9cb72b2a","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":321,"at":"2025-11-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-18","status":"stale"},"measurement_id":"e0ec5fa58fa1508ad2a1e0a663666dd5e7aac770069202124165ec82b782c67d","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"standard_error_points":2.4720163409,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2025-11-18","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"b65546bf4f0c88ae171747ccd7ebbf5b49d39cc61a6575952c58fdab43b785ed","result":{"confidence_high":61.69908461246399,"confidence_low":52.00878055613599,"sample_count":89,"score":56.85393258429999,"score_display":"56.9","source_stated_rank":60},"run_fingerprint":"43b026e2e5cffa88ae01086121f8da52b080929cef6309f3dd271ddf450e6b2f","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f06c5117dc5d721d9cb72b2a","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":321,"at":"2025-11-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-18","status":"stale"},"measurement_id":"35476e7bd7cc156946acc27f55bfc123d5911419a8ccf4869c8aa2f84e6323e8","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"standard_error_points":2.9709938903,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":"2025-11-18","source_scaffold":"Letta Code","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Letta Code","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"dc57c72a6ad7e1372a3f06339f47bd70400c04761f0759f96a126ee380ab2a93","result":{"confidence_high":61.778204204788004,"confidence_low":50.131908154812,"sample_count":89,"score":55.9550561798,"score_display":"56.0","source_stated_rank":61},"run_fingerprint":"85ac9291e3bd2e6cfb317e632eaa9bb6f0580bc32ca9cbe9f1359fa12f0970c5","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f06c5117dc5d721d9cb72b2a","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":217,"at":"2026-03-02","basis":"evaluation_at","evaluated_at":"2026-03-02","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"80b371991481f31c6225949ba3e8566d8f981287edd24d3a2325608dc313e2d4","metric_details":{"cost":null,"pass_2":9.28,"pass_3":5.67,"pass_4":4.12},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"terminal","submission_slug":"gemini-3-pro_sierra_2026-03-02","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"6070e5eb3b8465e8c6f5c8390e51640e8e9064c49cf8b5ad9ed333ecc2cba0e6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.04,"score_display":"18.0","source_stated_rank":null},"run_fingerprint":"0ffc618fbeec5ac27644700488a3501f544b5f7779d0a64f805751310b44ee6d","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_adb9d064fb6ec60affbc6752","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":290,"at":"2025-12-18T18:01:43Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-18T18:01:43Z","status":"stale"},"measurement_id":"954e22e5719130d3884e9e592f2d53e5e4bfd91a8e3d55dc0e3259c514fa1887","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.8,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gemini-3-pro-preview; createdAt=2025-12-18T18:01:43Z","source_row_created_at":"2025-12-18T18:01:43Z","task_pass_coverage_threshold_percent":75},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"gemini-3-pro-preview","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"6394a395f47cb539b8cffc20aa5f140a26427264a2766485300ebe7e47db6cd1","result":{"confidence_high":73.1,"confidence_low":67.5,"sample_count":1000,"score":70.3,"score_display":"70.3","source_stated_rank":13},"run_fingerprint":"32441c0cc15b850d68a5a8df626b781da2f32aed37464dc15bbb5aa6bdb1b908","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=gemini-3-pro-preview; createdAt=2025-12-18T18:01:43Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_671f7a505d0631492846b729","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"f27d304f470322614c3056caea7e3e3eca8692f84b68a2e6d5b5016e834672d0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.467","mean_standard_error":"3.4","model_release_date":"2025-11-18","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.6,"upstream_source_url":null},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"5bc556bcd20c203aad1c5f702bafe41cc432f3b4006544c8231054664940f2d2","result":{"confidence_high":38.556,"confidence_low":24.444,"sample_count":480,"score":31.5,"score_display":"31.5","source_stated_rank":27},"run_fingerprint":"2ec5fcac0d12efa6319bacae77fdc8dca94c783a0cec864c34c9b882239fd881","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e6bf48749d045f9009bebce5","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:42Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:42Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ce2a9ecb4b2ce21ab28a8d84f235cd24a24e419d12e1dc2ffd8eded987ba1652","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":15.87,"leaderboard_last_updated_at":"2026-04-12","license":"Proprietary","model_link":"https://deepmind.google/technologies/gemini/pro/","total_cost_usd":224.69},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"FC"},"run_count":null,"scaffold":"FC","tools_allowed":null},"protocol_fingerprint":"6a9c99bccbca11748bdf70a2bdd6eb864726c94bf4a029f84ab95898fe96d192","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.14,"score_display":"68.14","source_stated_rank":7},"run_fingerprint":"7f17039421980bb44eedcd74613dbbf5dd2682ba46d9b6e9cb1f9ddc7e9c1083","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e10ca9a1e04ebb6244b12dd1","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:03:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:03:18Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef1e412703ee718956c454641d3eaea46525aa2ec2bc0d515318f29114ba7da0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":12.08,"leaderboard_last_updated_at":"2026-04-12","license":"Proprietary","model_link":"https://deepmind.google/technologies/gemini/pro/","total_cost_usd":298.47},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prompted-function-calling","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"Prompt"},"run_count":null,"scaffold":"Prompt","tools_allowed":null},"protocol_fingerprint":"564fbea687964bbbeb4c2a4d92f977e2c00894a357d9ecf0f2b928b7737d37ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.51,"score_display":"72.51","source_stated_rank":3},"run_fingerprint":"1769f5bb534c4636ef624e04d6a0ca5e3f192cb11edd87303095a8f08eef4dd9","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e10ca9a1e04ebb6244b12dd1","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prompted-function-calling","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"652fe673e3a5b184bf81e58d7f94bd7c2ab6852a0dfdb8929c0d568ded7021ad","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"8c8835ad8aecbf44202e08a057c93f84eca18a816e9e74d6574afe97bfd73116","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5478.158000000001,"score_display":"5478.16","source_stated_rank":22},"run_fingerprint":"0069c91a34a8181a3452424f96eb3c4c41ec462ed126287f3f6dd59ed9004dde","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cc3ed16981114b07863ab4cc","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval","metric":"win_rate","name":"GDPval","release_id":"release_730527fa346470703bd19e21","split_or_window":"expert-comparison-win-rate","unit":"percent","version":"GDPval official leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:05Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:05Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"34c622943b63c4247dc75e660d7641327983532370ff454cc3bd2efe61387aa1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-18","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-pro","name":"Gemini 3 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Pro","source_effort":null,"source_model_version":"gemini-3-pro-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenAI GDPval expert-comparison protocol","tools_allowed":"as reported by the GDPval leaderboard"},"protocol_fingerprint":"2b572160dc5691512f728679f7c793b9cf2eb4c1ed163081ee67c969327580c8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.3,"score_display":"40.3","source_stated_rank":5},"run_fingerprint":"9378466970baa9cb3c123af8e8dd765e0a3527f05e761e12c3e2f7df1e149563","source":{"content_hash":"d4eac25e0863d96ee45cc545fc896e5667d7022da8b3bc3902533ed87fa8b836","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gdpval","id":"epoch-gdpval","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GDPval","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gdpval"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ea20236fd014b2198d3a19c8","provider_config":{"source_id":"epoch-gdpval","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"adbac22a429fa3255ff384adbe14f306ee52b164514f1784d3b087e2a30692f3","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-17"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Sonnet 4.6; model release: 2026-02-17","source_accessibility":"API access","source_model_name":"Claude Sonnet 4.6","source_model_release_date":"2026-02-17","source_model_versions":["claude-sonnet-4-6","claude-sonnet-4-6_16K","claude-sonnet-4-6_high","claude-sonnet-4-6_max","claude-sonnet-4-6_medium","claude-sonnet-4-6_unknown"],"source_reasoning_efforts":["medium","high","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5eb08fc5d3fc08f08a0c05bad1875828dd5c488b832c0e43a9db9ee62aa9545d","result":{"confidence_high":154.69,"confidence_low":149.61,"sample_count":null,"score":152.26,"score_display":"152.26","source_stated_rank":40},"run_fingerprint":"2a8d5f49c6d8dc2e8fb95ff40d1e6cd2e17e047b5826cf0d3d7009ccf5c69507","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_da95b541a68b4f050b9475da","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a512d1483d895186acefa3e157762d6b158da6816912d7e21760b2cac43b7f74","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-02-17","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · intelligenceIndex"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-4-6-non-reasoning-low-effort","upstream_model_label":"Claude Sonnet 4.6 (Non-reasoning, Low Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"429a95c8f665fdc7c3a54c0033e86fb49fb5eef4d1b5854f4d6f2339bef9fc31","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":23.303501,"score_display":"23.3","source_stated_rank":null},"run_fingerprint":"328346c3d4bad2dc867a1822321a4b4822d2e43808a1c668784d8c07f9ee647b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e0bdc98a9afa7f9145f07deb","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0b63f25f88790c63509ee47af7d073edb3c71eb4defbcab4867c810ed40ed28f","metric_details":{"license":"Proprietary","variance":3.2273014058312675},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1475.6715602605889,"confidence_low":1468.629530595082,"sample_count":70747,"score":1472.1505454278356,"score_display":"1472","source_stated_rank":43},"run_fingerprint":"917cddfc5dd7b83e3c4592a40aa93e7fa5f8f055185a1dc8ac5f567f8b1a294c","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5f4b5142e38dcec1edde2831","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"17fec73292cf131953ba7c4f7ddba7652aa7d5640f12695f71d0e5b39628827e","metric_details":{"category_scores":{"Agentic Coding":42.626,"Coding":79.2715,"Data Analysis":77.946,"IF":63.22075,"Language":76.10266666666666,"Mathematics":86.994,"Reasoning":84.76925}},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.99002380952382,"score_display":"72.99","source_stated_rank":48},"run_fingerprint":"62ad7e0f4f0710a934176c820b63544d3f69ae33f51c7a45e1a196bcf5df0ff8","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_02f18483087dad439956139b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_c1afb6c54b8ceec8bc1f121c","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"ff5469580f4c8a133ad1cde008b35a24d9aa88e1ea5e12550f416c0053755308","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Table 8.1, p.115; Figure 8.9.1.A, p.122"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; effort level not stated; 1M total-token cap; no tools; no context compaction","question_count":2500,"reasoning_effort":"reported","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f3369573ec5fa708c2fb1887738ca58a6da3b6f6fdd4087151c44904e7e0c723","result":{"confidence_high":36.487064,"confidence_low":32.760192,"sample_count":2500,"score":34.6,"score_display":"34.6","source_stated_rank":null},"run_fingerprint":"45041b8488488b3f51a0825fb39be6084ad10a2ce27b301d8c35e49eb1377cfa","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Table 8.1, p.115; Figure 8.9.1.A, p.122","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1a899ff2f2c99f660795bf87","provider_config":{"source_id":"anthropic-sonnet-5-hle","source_label":"reported","thinking_strategy":"adaptive"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":230,"at":"2026-02-17","basis":"evaluation_at","evaluated_at":"2026-02-17","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0c261587e5cc498390cd689c9a20e6938cead5746cdfc4d89ac517bc30555412","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/anthropic-claude-sonnet-4-6","source_model":"Sonnet 4.6"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c7930601b65d3ec96975e60f5fc5805013f5fec9fece95a1dfc134c568e38f7b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":21.07,"score_display":"21.07","source_stated_rank":39},"run_fingerprint":"17cd75e29ba68a0839c1b945091d0390f083bf63a4c88003a433be0cc63c9470","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0c4826b710c73a7aaf93a4a6","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"c3a666760a149a3348135ece27cf588cfb8fafa52c49033e086f99cbf34c3500","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","provider_default_effort":"high","question_count":2500,"reasoning_effort":"max","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"7c3f637872fb46244f5b5da426d161cf1d53e48b97d141c1141f672acb42d4a9","result":{"confidence_high":48.759397,"confidence_low":44.850423,"sample_count":2500,"score":46.8,"score_display":"46.8","source_stated_rank":null},"run_fingerprint":"cb54539d60ec42b068d57b2a183276cff66991a5a9f19ace9cc5c7333d69c152","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_6e5545f77fade6f34977fde9","provider_config":{"source_label":"max","thinking_strategy":"adaptive"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"2dcc19faca3a183d2f75e8b24cb4cfef5e1e0375a0d8dba3e02feb248b160b60","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","provider_default_effort":"high","question_count":2500,"reasoning_effort":"high","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"aa806c655efba8a3459511fb2c31f04999569fd04d0880d3a6f1a4f22f37520c","result":{"confidence_high":51.658921,"confidence_low":47.742,"sample_count":2500,"score":49.7,"score_display":"49.7","source_stated_rank":null},"run_fingerprint":"500c0a61750071de49662ddb62da4ac732e1ab82a4dd84de5334993b2454a659","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_19e9c3be0c0df85e76c55fcb","provider_config":{"source_label":"high","thinking_strategy":"adaptive"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"e72c6a335cd7c95e48b36d140d2cfbed127e45842d30d04bd33342c30a2d28f8","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","provider_default_effort":"high","question_count":2500,"reasoning_effort":"medium","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"e6ac5a2f2a68f1723a6925f4b9c68204fc7c931eafd2b4094cadc3e54296bd51","result":{"confidence_high":48.459069,"confidence_low":44.551671,"sample_count":2500,"score":46.5,"score_display":"46.5","source_stated_rank":null},"run_fingerprint":"da0b63fe36c519c56dede8bf2ea8dfdbd956c0b50ac99738f35cc8f4cdd1b3ad","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_53137800ea2a4a698cd72c42","provider_config":{"source_label":"medium","thinking_strategy":"adaptive"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_d4c0348f1b08f65a1ad85f5a","split_or_window":"full-multimodal-2500-with-tools","unit":"percent","version":"HLE original full multimodal"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"440c0f09b319619028c4208ce410e5130c5f7e7e60f69dae67fc6055f291d1fb","metric_details":{"comparison_warning":"HLE full multimodal with tools; kept separate from the no-tools comparison lane.","comparison_warning_code":"hle-tools-vs-no-tools","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Figure 8.9.1.B, p.123"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-tools-vs-no-tools","judge":"updated vendor grader","run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"adaptive thinking; stated effort level; 1M total-token cap; no context compaction","provider_default_effort":"high","question_count":2500,"reasoning_effort":"low","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"web search, web fetch, code execution and programmatic tool calling"},"run_count":null,"scaffold":null,"tools_allowed":"web search, web fetch, code execution and programmatic tool calling"},"protocol_fingerprint":"a1828867c18970142823c0f667c112f937bc0b7c190df942b85741fac6dfc1d8","result":{"confidence_high":36.790361,"confidence_low":33.055974,"sample_count":2500,"score":34.9,"score_display":"34.9","source_stated_rank":null},"run_fingerprint":"df480c0d9bd43947167f228d4044730750945d85c0e600c5375f4b2fe83a2994","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Figure 8.9.1.B, p.123","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_de3784cde67f5c2f86e15f04","provider_config":{"source_label":"low","thinking_strategy":"adaptive"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c7e4b371b86361ce3b5d7d238caeac0b60b76334af44e299c6bd48f5324b20c7","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-02-17","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · hle"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-4-6-non-reasoning-low-effort","upstream_model_label":"Claude Sonnet 4.6 (Non-reasoning, Low Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b982a6b744e7fd55a5a61ca3b7e90f01a548066ccee70340d09bdca8af5787fa","result":{"confidence_high":12.614937,"confidence_low":9.951083,"sample_count":2158,"score":11.214087,"score_display":"11.2","source_stated_rank":null},"run_fingerprint":"d8098d508d4db58dc96463124829d48b7c35450809c3bac6d8031e0ab3c50235","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e0bdc98a9afa7f9145f07deb","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:27:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:27:37.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"09eb3c7a34ee3a72062f3b0b64e3e36989a0fea6e2bfcddfd26da6ed6eee7daf","metric_details":{"best_score_across_scorers":"0.328","model_release_date":"2026-02-17","stderr":1.4853842487270312},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Zs5fCvNzaz8LhqgwDSAztU","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZs5fCvNzaz8LhqgwDSAztU.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Zs5fCvNzaz8LhqgwDSAztU.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"b0e6d3b8121dc6f9eb2cae0167640e4069ac1148c0d8cacb2a79d747591e72ee","result":{"confidence_high":35.711353127504985,"confidence_low":29.888646872495023,"sample_count":1000,"score":32.800000000000004,"score_display":"32.8","source_stated_rank":60},"run_fingerprint":"91cb93f825df36e466c79c61a544e15cc5db2fde6d3752f3ddb6cfba1a9e1786","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T22:42:23.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T22:42:23.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d7d26430d9cbcecc53a09425864a6f0f8eef8181b1dc9d1f22b23d4cf6d5361f","metric_details":{"best_score_across_scorers":"0.355","model_release_date":"2026-02-17","stderr":1.5139491543780594},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LJyKf8Xy4dymxNHbGLwNdA","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d781b43e7f6d23901543cdc8b0bf96fd38f2e0bdca083e04774ac3c7a4d6429b","result":{"confidence_high":38.467340342581,"confidence_low":32.532659657419,"sample_count":1000,"score":35.5,"score_display":"35.5","source_stated_rank":50},"run_fingerprint":"f42b6666894d7a8348d94ca8b4287233f836994a81398a01f41b58c45b593dc4","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":209,"at":"2026-03-10T00:00:00Z","basis":"evaluation_at","evaluated_at":"2026-03-10T00:00:00Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f877d74e37045d0794a43b2e22472c883e85b630ab5f083c2929169e19b30e38","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-sonnet-4-6-default","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8fde30ba473282f550b181c561072188f2351fd6bd49261cf4073a6e1d86fa4a","result":{"confidence_high":33.925900736144385,"confidence_low":27.84193738952119,"sample_count":null,"score":30.88391906283279,"score_display":"30.9","source_stated_rank":24},"run_fingerprint":"2c8cb3080da5110c5969ee351ebdf18ab54b210662f29bf7c8b679a53aa119ec","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bcf8723a7f7b20582b88aa73","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"43c02fc9434e303a4250945484919bf5572a8a4370de129f8046bf17d995d041","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.408666,"date_is_proxy":true,"latency_seconds":377.864,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.406},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","few_shot_accuracy":85.859,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":true,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-6","zero_shot_accuracy":85.354},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"8824b5cfaea52673f457e5b1ae766e9ddc542ff5c373221885ef8f6537ea2186","result":{"confidence_high":90.32176,"confidence_low":80.89023999999999,"sample_count":396,"score":85.606,"score_display":"85.61","source_stated_rank":46},"run_fingerprint":"6f9b9af63a719d91df57bf14347190e24cd7fb504abdb4b839ab3a7a6bedb4ce","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"95b80f5ab60518b769643991d12cae423aa6240a3cbb7889b1fc93470e1f79c4","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-02-17","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · gpqa"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-4-6-non-reasoning-low-effort","upstream_model_label":"Claude Sonnet 4.6 (Non-reasoning, Low Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"02c7269d1b41441aa5e3db91d3ba3badbadce1144ff321feeaeffbdf64d9ccd2","result":{"confidence_high":84.709942,"confidence_low":73.553569,"sample_count":198,"score":79.69697,"score_display":"79.7","source_stated_rank":null},"run_fingerprint":"1b8649d74d172dd257de148ae53af3dadf9f41101f5e27e7980d10d8c03da954","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e0bdc98a9afa7f9145f07deb","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:21:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:09.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6cf8ed1c18dee0fbbac970a78393293bdfbf189387bf6e758158f9be7e117b05","metric_details":{"best_score_across_scorers":"0.7878787878787878","model_release_date":"2026-02-17","stderr":2.9126522834586774},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"SEePRKRcv8oqn5HpD2KFxB","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSEePRKRcv8oqn5HpD2KFxB.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/SEePRKRcv8oqn5HpD2KFxB.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"00d5cdb97a6f04832c233b81aaa20dd83a92f85e2fa37cf44eaea8247bf12bcf","result":{"confidence_high":84.49667726345778,"confidence_low":73.07908031229978,"sample_count":null,"score":78.78787878787878,"score_display":"78.8","source_stated_rank":133},"run_fingerprint":"c6591a987983e7d95b8baffdd7a006792341b91f68609864c81b53f2c61704c5","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":83,"at":"2026-07-13T17:28:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T17:28:15.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5b8f2488c822305467885836add139be236304964d775381cf80b57841a9368d","metric_details":{"best_score_across_scorers":"0.8333333333333334","model_release_date":"2026-02-17","stderr":2.6552207828215257},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"DmdKPD9Z9RXwYyATAoG4Uw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDmdKPD9Z9RXwYyATAoG4Uw.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/DmdKPD9Z9RXwYyATAoG4Uw.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"05c07b4b5440cf26bc9a67a50ff4afa4cb477f4690d813653a5b0fc3799891d2","result":{"confidence_high":88.53756606766353,"confidence_low":78.12910059900315,"sample_count":null,"score":83.33333333333334,"score_display":"83.3","source_stated_rank":104},"run_fingerprint":"bfdb1a0de30df4905706d104928c9594333205c5f2ecd3f135a3ede8e8ee00ac","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":83,"at":"2026-07-13T17:28:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T17:28:00.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"efc475ae68c37599c59ddcee0e5cb243750edbb4c755abfeea39ee7741ff778e","metric_details":{"best_score_across_scorers":"0.8333333333333334","model_release_date":"2026-02-17","stderr":2.6552207828215257},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"e8MXnAM8HMF8eEzYtYA7xv","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fe8MXnAM8HMF8eEzYtYA7xv.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/e8MXnAM8HMF8eEzYtYA7xv.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"dc11e80272bfbf7199866bc4e945e7be1da01b38a5ce0096e4dd0a2655d07f41","result":{"confidence_high":88.53756606766353,"confidence_low":78.12910059900315,"sample_count":null,"score":83.33333333333334,"score_display":"83.3","source_stated_rank":104},"run_fingerprint":"6ff2537c83253044921721b6091bd2a462f261e3febadf2a152a19e1100f2275","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_02f18483087dad439956139b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"235d4e0dc4b48d5a06940de12bf7d5c05dd6fc19219560673d16840eb4607894","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.066412,"date_is_proxy":true,"latency_seconds":55.406,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.434},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":true,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-6"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4127eaf4a50234957890d224b1ca7a0606a7f1214b342d970a2750ed4228dcbe","result":{"confidence_high":88.19163999999999,"confidence_low":86.49036,"sample_count":null,"score":87.341,"score_display":"87.3","source_stated_rank":29},"run_fingerprint":"e394a9b91cd02cf3b5653640bb2bed15aedde5dc9686675c3f37ae89d1f529d1","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:21:16.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:16.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4b48ea84eddf53a00fb54a81e4ea82f68b195042f6d9535378e4fb69d1d63b3e","metric_details":{"best_score_across_scorers":"0.7111111111111111","model_release_date":"2026-02-17","stderr":6.832943242540508},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"XnQVH5qSfxVjpzoPwUEmnh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXnQVH5qSfxVjpzoPwUEmnh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/XnQVH5qSfxVjpzoPwUEmnh.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0f46c770def16a961a1818eaf4b929b42f199ee5e3b9864469ec01a2413995cd","result":{"confidence_high":84.50367986649051,"confidence_low":57.71854235573172,"sample_count":45,"score":71.11111111111111,"score_display":"71.1","source_stated_rank":132},"run_fingerprint":"cbcf7bfbd344cd3fa7791d8db78058602dbb20ef0f06b3ebb9141b382a8cdf62","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T17:28:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T17:28:15.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"206833860354ad183cf79f290240164f4fa4c8bc5a797f6922c9c7c292b1a06a","metric_details":{"best_score_across_scorers":"0.7555555555555555","model_release_date":"2026-02-17","stderr":6.478835438717},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"oXZ7ZVZtADrwFPKAe6zTvL","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoXZ7ZVZtADrwFPKAe6zTvL.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/oXZ7ZVZtADrwFPKAe6zTvL.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b90b5d51d7f11694bcec286e27edfd26460492950d3ba3dd7af5e1b467a37004","result":{"confidence_high":88.25407301544088,"confidence_low":62.857038095670234,"sample_count":45,"score":75.55555555555556,"score_display":"75.6","source_stated_rank":123},"run_fingerprint":"ec76f291ab4c21eb2869841f86e5e2955626639b726a3c07e40ee1e6bc31c64e","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T17:28:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T17:28:00.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"592758469c0d5c090a779da1837f3db948073cd5228794119ce51bb172b234fd","metric_details":{"best_score_across_scorers":"0.8222222222222222","model_release_date":"2026-02-17","stderr":5.763774795025092},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"kDNNbUYXhrAH5uDBGykFaf","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkDNNbUYXhrAH5uDBGykFaf.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/kDNNbUYXhrAH5uDBGykFaf.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f0bf4961d36b31acdd04b23261eb59de0ec4597d4be487a3fe7bbe059171d5d9","result":{"confidence_high":93.51922082047139,"confidence_low":70.92522362397304,"sample_count":45,"score":82.22222222222221,"score_display":"82.2","source_stated_rank":103},"run_fingerprint":"a232f815d7d03c0007a11d97e516e0a66b4af7c9d6d03372e221355a112189a3","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_02f18483087dad439956139b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7c33273fa91e47052dc66abb1c8225253c7df8a3dbf2a260948a3d1b54bec809","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":2.7179,"model_release_date":"2026-02-17T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-sonnet-4-6","model_type":"CoT","upstream_model_id":"claude_sonnet_4_6_max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"11a379dad5a9bd83ac82b0619340f7e3af4d02aada4a2643ff8f4053b46d0328","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.330000000000005,"score_display":"58","source_stated_rank":83},"run_fingerprint":"2aa5d1a77b3f26e6b154f3b33b61ce82e8e4b5de47fec8d9a92eece2d417ebe3","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0b33bd0c0b63716c3fb557a8e1e97f41d386ba3a6f3cb469162fa790351cf2ea","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":2.6966,"model_release_date":"2026-02-17T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-sonnet-4-6","model_type":"CoT","upstream_model_id":"claude_sonnet_4_6_high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4c34066be47bcde40116e0831c90bb7f4af59de9d176c5f85241adee287e3f3c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.419999999999995,"score_display":"60","source_stated_rank":75},"run_fingerprint":"3c9509597e8b8d80a177cd219a9d59846dfbf3ac2565926ecda32b2d19beb47b","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dee6a1cb86321608cfec15d3f53b1f346a3b0f4f3581c29cd0d67760f7a343f6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"2.7179","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6 (Max)","source_effort":null,"source_model_version":"claude-sonnet-4-6_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a8b291804a9702356151ec54f311de44ef06bebba92e67e0db3969f82a12e088","result":{"confidence_high":63.308724,"confidence_low":53.175373,"sample_count":360,"score":58.330000000000005,"score_display":"58.3","source_stated_rank":85},"run_fingerprint":"a8a066812f17237197e9dca5177b0fc2fc8363c3fe5fe40c52ab92dae09e09f2","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bf3102b352e3ef16170f4a92e18a1c9a7b568e04f96ce18cd85fe97ba460c616","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"2.6966","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6 (High)","source_effort":null,"source_model_version":"claude-sonnet-4-6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"efe665b6dc29940e343ccfb986dde3ce47c4543fe64831ab5949dd9f45001303","result":{"confidence_high":65.336094,"confidence_low":55.283868,"sample_count":360,"score":60.419999999999995,"score_display":"60.4","source_stated_rank":78},"run_fingerprint":"8575e54d4820a66eedafcf9d0928a5c166e1714a246199e5c8a10ee98d822a94","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3eb5841aed22ae1549078b3e999e1e3ab16372b778f397aa876fbaa63357597d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.144122,"date_is_proxy":true,"latency_seconds":131.728,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.891},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":true,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-6"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6f43ce4955f6243fc835b09ef61a635f340ba3565b86367bf06f36a7ef3d49df","result":{"confidence_high":85.33036,"confidence_low":81.83764000000001,"sample_count":null,"score":83.584,"score_display":"83.6","source_stated_rank":30},"run_fingerprint":"a9589e9a5bd8a2b8a2425409689e9707a8d8c4a7631750b56a908e21de86f562","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"593afd0e27070f78d66971f114ca98fee913dab535ba71b2a45083359443ebc7","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-02-17","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · mmmuPro"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-4-6-non-reasoning-low-effort","upstream_model_label":"Claude Sonnet 4.6 (Non-reasoning, Low Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"13f558c16e71bbff80e8516e3166ee9a5270a049341658cd9242cb682db82867","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.190751,"score_display":"69.2","source_stated_rank":null},"run_fingerprint":"e646dcd578f4eecb57386e206923bf68fe3812f3309405eb8e83842500be9602","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e0bdc98a9afa7f9145f07deb","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b8b11a2c68fe6897b0cdfde597e940271bc787896bdbeff60acb1f9f41382aa5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-sonnet-4-6_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8b32a603bf213ff2effecc0233c59309cbb61896453b8e1ce7276f9e9c96348b","result":{"confidence_high":10.175118,"confidence_low":0.920927,"sample_count":71,"score":3.14285714285714,"score_display":"3.1","source_stated_rank":121},"run_fingerprint":"d746a488164095006286b2abc49a2a828846d3f0c11e3289aa71fe563284bbfd","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"396a2d47c8ac23d45340caddec5b97003eb6af5577f98cc8f1b8a03077cf18d0","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-02-17","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · critpt"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"claude-sonnet-4-6-non-reasoning-low-effort","upstream_model_label":"Claude Sonnet 4.6 (Non-reasoning, Low Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a5b147e0e3ace6b42d6a12d624727187fa8b751251905e2567ddc62efd7d79c0","result":{"confidence_high":6.724008,"confidence_low":0.10358,"sample_count":70,"score":0.857143,"score_display":"0.9","source_stated_rank":null},"run_fingerprint":"9bc907a4d2f34dc58209f40923b9c18b12a8953c9ae5b76c341c6fa690fb7bf8","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 4.6 (Non-reasoning, Low Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e0bdc98a9afa7f9145f07deb","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:21:10.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:10.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7254e0a500e3c6ccc94833abb782c332c9eb63832dac4f0bb2f9f93f34a1aa28","metric_details":{"best_score_across_scorers":"0.03","model_release_date":"2026-02-17","stderr":1.7144660799776528},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"TDT98jZXHbUMFYTphGEpqC","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTDT98jZXHbUMFYTphGEpqC.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/TDT98jZXHbUMFYTphGEpqC.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9bf047600afe6493a7a8a9fc5c6822c81fe8f2d17cde3809323b3b247d613b23","result":{"confidence_high":6.3603535167561995,"confidence_low":0.0,"sample_count":100,"score":3.0,"score_display":"3.0","source_stated_rank":180},"run_fingerprint":"fd97c1d30461bb80d2ed05cf5ea8303306369a56cbd4aad0965376819e406925","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T17:28:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T17:28:15.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"129ff07bb96e9338aa962c512d8fb3e2c403448f1690eb640b9d2e832af6ae95","metric_details":{"best_score_across_scorers":"0.05","model_release_date":"2026-02-17","stderr":2.1904291355759025},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"5Q9RNC2JfEmmvNFFE5CQCf","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5Q9RNC2JfEmmvNFFE5CQCf.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/5Q9RNC2JfEmmvNFFE5CQCf.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0cf137a82047241ad1aefa6461b2e876c5045122dc1ae7abdf0f21447622be1a","result":{"confidence_high":9.293241105728768,"confidence_low":0.7067588942712311,"sample_count":100,"score":5.0,"score_display":"5.0","source_stated_rank":161},"run_fingerprint":"1a6aa360487b2f0febee73202c441e5f5e6525b0294f4416df5d3ae9b3d3b997","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T17:28:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T17:28:00.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a0a84c849109fd035f8ee546af77603d04b10d0edc2701927fdfb7d41c5ab99e","metric_details":{"best_score_across_scorers":"0.08","model_release_date":"2026-02-17","stderr":2.7265992434429074},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"eJBzZyT5ohb8ieaJCum5rK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeJBzZyT5ohb8ieaJCum5rK.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/eJBzZyT5ohb8ieaJCum5rK.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a89aa532004456ab7cdcdd3bddd4146eb1f2e2831766eca08ec51714c19bc30a","result":{"confidence_high":13.344134517148099,"confidence_low":2.655865482851902,"sample_count":100,"score":8.0,"score_display":"8.0","source_stated_rank":145},"run_fingerprint":"970595ed284f9ae37b1f4f5a438a78d0a4cd3c20bd82025e5aebf9390e42190c","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_02f18483087dad439956139b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c8033213ccf4e2c64240227239814cc0b098abed44fd71db84feb1165901b659","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.452","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":""},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6 (Max)","source_effort":null,"source_model_version":"claude-sonnet-4-6_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"1524582ed693ccee7f08afa2bb1f91e502bc548f5f232eddfb3f196bda91939c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.0,"score_display":"86.0","source_stated_rank":99},"run_fingerprint":"f4a2cb983fcb3b5763d3948545f47b4fe4d21f8ae8f48417d965e66e1a51d3ea","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"345b27c1c98096b8dd3ec00d0291b9f1129e0641fec9e45773e9743b5ab05dc5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.1908","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":""},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6 (High)","source_effort":null,"source_model_version":"claude-sonnet-4-6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"eec758442b533649a580f224111cd881f95defbca7a9bc67c3ca033947fdb532","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.5,"score_display":"86.5","source_stated_rank":94},"run_fingerprint":"cfe187418da39929d069ec7e14b436ad19b140f7ae58f372d872aa7b5327d65d","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"44b350242c74e3412985291d4c637f52f5e9589d8c276c46befff52acf93e29d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.303853,"date_is_proxy":true,"latency_seconds":511.785,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.872},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":true,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-6"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"7cfa6bd9fdb868ee8d5ddf5b9739fb3c5d468cb688fd433b349205e469c8a7fe","result":{"confidence_high":81.06912000000001,"confidence_low":73.73088,"sample_count":500,"score":77.4,"score_display":"77.4","source_stated_rank":35},"run_fingerprint":"0ac3cd634104c957df61b18350ac2cf1f2678025f64bc2708a9f8c630ced3989","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":226,"at":"2026-02-21T10:32:51.288Z","basis":"evaluation_started_at","evaluated_at":"2026-02-21T10:32:51.288Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a8450269c94824ea7d583da9eb23194ca01d71c89d014c308b523613f90991c2","metric_details":{"best_score_across_scorers":"0.7520661157024794","model_release_date":"2026-02-17","stderr":1.9648182701390322},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"KEdgiMmtJKHxgGiS82agYj","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKEdgiMmtJKHxgGiS82agYj.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/KEdgiMmtJKHxgGiS82agYj.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"1a888b334b2a7a415fcc15fa35b63f84e79427c9581182fd2b6c55e1f99ad5ca","result":{"confidence_high":79.05765537972044,"confidence_low":71.35556776077544,"sample_count":484,"score":75.20661157024794,"score_display":"75.2","source_stated_rank":15},"run_fingerprint":"c22a2e44c8ca1d9407d9d94be2074b3319cae343661a3fb5a23ddff654205722","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_864231988819cafbf350efaf","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"8d5c6403f38b12208106d7a4460c0b870b1caede8d7fcd6ecc9539cc5229104b","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-02-17","pass_at_5":63.63636363636363,"potential_contamination":false,"sem":0.5454545454545447},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"Claude Sonnet 4.6__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"690f71bd7ea1ef47ffbe550903858995f642b0efe28b1a7c4ea90de3356334b9","result":{"confidence_high":52.34181818181818,"confidence_low":50.203636363636356,"sample_count":null,"score":51.272727272727266,"score_display":"51.27","source_stated_rank":null},"run_fingerprint":"d5bd5ba287f21671b93a9c7a9d584cdccc472474131a2a2b0f115faf67c5facc","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e6ae4e8ef0b5692fde876ef0","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"165d363bb652874fea00c9e143c951ca2e1a6450a3fa20407b50794f7b4fd176","metric_details":{"ci_attempted":451,"ci_half_points":4.092141269189136,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":135,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":133.65853658536585,"mean_cache_tokens":12719994.494456762,"mean_cost_usd":5.5223517987804875,"mean_duration_seconds":2815.01307943459,"mean_input_tokens":12870627.738359202,"mean_output_tokens":76160.31263858093,"median_agent_steps":124.0,"median_cost_usd":4.867440150000001,"median_duration_seconds":2373.991057,"median_input_tokens":10933709.0,"median_output_tokens":70001.0,"median_output_tokens_to_pass":67060.0,"median_peak_context_tokens":146430.0,"n_attempted":451,"n_passed":135,"n_tasks_attempted":113,"n_tasks_passed_any":64,"pass_at_4_points":56.63716814159292,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_claude_sonnet_4_6_high","source_model_version":"claude-sonnet-4-6","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"ada28153a47df2486f049859a8052f5c5e8be46e72fd73892686844c49cb15d6","result":{"confidence_high":34.02562242218249,"confidence_low":25.84133988380421,"sample_count":451,"score":29.933481152993345,"score_display":"29.9","source_stated_rank":65},"run_fingerprint":"da795be427acf83b346f70b1f2dbb787623e71a42c3e8aeec4627c3bdaa176e9","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"beb72b32f8de5d10c7c4363d6de23ecde78910f57f4b3faa201c320e06914a7e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"133.65853658536585","mean_cost_usd":"5.5223517987804875","mean_output_tokens":"76160.31263858093","model_release_date":"2026-02-17","notes":null,"pass_4":"0.5663716814159292","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-sonnet-4-6 (high)","source_effort":"high","source_model_version":"claude-sonnet-4-6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"6141e7bbb82143877ca22af0ff0544cf4045116fe3718babf5f0ff22d5380b48","result":{"confidence_high":34.02562242218248,"confidence_low":25.84133988380421,"sample_count":113,"score":29.933481152993345,"score_display":"29.9","source_stated_rank":64},"run_fingerprint":"bac946d6ded13a938feb0ad58a9d25c93a9fd532ee5d6c37e81d515bd256bbad","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_08c854386a3f959f46602e62","split_or_window":"vendor-public-own-harness-comparison","unit":"percent","version":"SWE-bench Pro · Sonnet 5 comparison row"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":97,"at":"2026-06-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T21:07:54Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-30","status":"stale"},"measurement_id":"f93abc7dbb04e7a2d0d5f5444d030693d77d2d398d2199dabab640a1114a0040","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Capability summary and §8.2, SWE-bench Pro comparison"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Anthropic","evidence_verified":true,"freshness_proxy":true,"protocol":"Anthropic comparison row; exact effort, original run date and run count not restated","provider_default_effort":"high","reasoning_effort":"reported","result_published_at":"2026-06-30","source_content_hash_method":"raw_bytes","source_content_sha256":"33573adb9f1871b903f79b77d7755a44cc20bb913ef48ac9000ddb090e41f7ed","source_published_at":"2026-06-30","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Anthropic agentic harness","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"09a0310dc395a66afe72d1794b0841af1298322806a4f4b36783402c28b01db9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.1,"score_display":"58.1","source_stated_rank":null},"run_fingerprint":"edbc28574d300b1dfa62c81369f799cb2b2da6733d225710098726ed31d5103b","source":{"content_hash":"9e00983df8055e10b62b8940b6fb13d5c0c4ff4ca13e379d7901201b7a9370d4","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-02T12:19:29Z","homepage_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf","id":"anthropic-sonnet-5-hle","last_fetched_at":"2026-10-05T12:33:34Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Capability summary and §8.2, SWE-bench Pro comparison","name":"Anthropic · Claude Sonnet 5 System Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_07047a630af9de9825b3077c","provider_config":{"source_id":"anthropic-sonnet-5-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":4,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"secondary-re-report","verified_status":"secondary-re-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"3644735d96df887e09215b3fd70fc9034c3ef3e32d6ad5a03ca582882dd79d11","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"3.09","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-02-17","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"55","tokens_per_task":"40280","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 4.6 Max","source_effort":"Max","source_model_version":"claude-sonnet-4-6_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"be1378564bfe56145f09d8a56be8c15e160e3be686f33e96471fea692b7efeaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.0,"score_display":"49.0","source_stated_rank":63},"run_fingerprint":"9f86626062d1523196fd173a9086c3916a17daa3ae4011be03acbe18dbe01748","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"18c783958274093eed1196983681ba0152bb8df9e32f82f4c5c7d02cf2dd2b70","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"3.06","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-02-17","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"57","tokens_per_task":"37352","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 4.6 High","source_effort":"High","source_model_version":"claude-sonnet-4-6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"7cf5aaea01f92425da3ce772fb49116175eebf0667d07fc10c07ee8060a3adbe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.8,"score_display":"48.8","source_stated_rank":64},"run_fingerprint":"be689be2510560745c09d8724c908199aa13af6cbf78597768b4b3111bd34526","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"78019fa78f49bc3f0b612c3beacaf5bfe1f95d64b260ca204076e07e1cd19fd0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"2.64","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-02-17","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"50","tokens_per_task":"31360","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 4.6 Medium","source_effort":"Medium","source_model_version":"claude-sonnet-4-6_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"a2bcc7863600186c288348769126cf01f7777489617632d1deea5cbd3fe8c3b6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.0,"score_display":"46.0","source_stated_rank":72},"run_fingerprint":"217ec0e6e464152acbd170ea3d26c9413ce584e1d3f3579a95bc891255225e1e","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_02f18483087dad439956139b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a94bf1830611ad3abd58fe75bb080ce0ddcb938864b9993c5979d690ce022f4a","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6","source_effort":"max","source_model_version":"claude-sonnet-4-6_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"claude-code","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"363f5ff210e51b2137fdcc9eb345984fc28f27ee9565cbde91a4859553167d2f","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":24.310000000000002,"score_display":"24.3","source_stated_rank":33},"run_fingerprint":"8c57c9fe539c6a832b049726300275ea3ee20717db3dc0a8cf262b43c5fa9cf7","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7eb0b914d3aafa55c065eac25e0887c374b94fcd4ad624c6779a395454e513d3","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.370575,"date_is_proxy":true,"latency_seconds":295.009,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.063},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":true,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-6"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"c99bbd3f4346cdcd9b0fbcf9a056f55180494d7e2fc8e7be949ef1d9a7c46be0","result":{"confidence_high":84.17447999999999,"confidence_low":80.00752,"sample_count":null,"score":82.091,"score_display":"82.1","source_stated_rank":57},"run_fingerprint":"631f3b0a1833aaa4e00a89e22b8958df94b15318050198b512b0464d4d2287c7","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"152bc87a6f2cd6ecfe39dda2c523f21cfcd0951ef7ae571e16f3493d162915e6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)","source_effort":null,"source_model_version":"claude-sonnet-4-6_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"efb8fe7fec2a2450eac8fa367c73e99ae36b1f2644c728fa4cad58d2a1ab1c4f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.7592592592593,"score_display":"46.8","source_stated_rank":116},"run_fingerprint":"3d48cc06cfd3fa176bd951d7574039b68d0c6a1dcd3c21348094b077af2bfb87","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0196532e2255b283f8f2560f002aac5cd45c26ff1c511e3f63edb02b189aaf20","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"1.770753","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"claude-4.6-sonnet-medium","source_effort":null,"source_model_version":"claude-sonnet-4-6_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"ca66496b8f55b81a101ae4184d534d5bfc193662ff3fab9d6667fef5f94674e5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1327.3,"score_display":"1327.30","source_stated_rank":24},"run_fingerprint":"e8ee7265899e21ce66d6c7e4264224e90cf91e9e9b851a1f7308f0cd60ee1bec","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_02f18483087dad439956139b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"34b1c3af928e6d0de70a455b597105bf6a17af2a154ae54c4a19d787390d4498","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":5.907892,"date_is_proxy":true,"latency_seconds":1572.118,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.645},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":true,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-6"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"29433fff21883c0014ce9616dbf8c6d502a5d01299aa34f35bd424accefbab37","result":{"confidence_high":60.5802,"confidence_low":42.3718,"sample_count":null,"score":51.476,"score_display":"51.5","source_stated_rank":50},"run_fingerprint":"1bd17b8406bd8010b66e316c578dafd864605e0713662d1f090fd24fe92276f2","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"012e4c05da22f7059ffad774ca5c4e35cc43044850eaa225f028bb44cd8705f5","metric_details":{"license":"Proprietary","variance":6.85785425935862},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-sonnet-4-6","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2a5775599154be0f95068709ca5852e7b375959c6a7e97675eec123f51831d8a","result":{"confidence_high":1526.5663701667204,"confidence_low":1516.301056779269,"sample_count":22960,"score":1521.4337134729947,"score_display":"1521","source_stated_rank":45},"run_fingerprint":"40a9e48a3ec4b08c08354d1c4d73d163c34b72029734e9046a2c1797c8c93b26","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fdb9efd11c7d6668480eb672","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"707fda42e6e65d2df29cf9fc0d930025d0b89a1caa441fd1e434829246f2c2de","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-4-6","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"b916ab1bb5cd21ae9ff395ce086a6bdb7aa7667853fe83790729a196ec244a8b","result":{"confidence_high":1526.24,"confidence_low":1515.73,"sample_count":21642,"score":1520.99,"score_display":"1520.99","source_stated_rank":44},"run_fingerprint":"2eb02ec728654dc673b0b68fef981502f44e5e83f052a3fd63f5eeb46e30a6c4","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_08c692d810974b560dd7f754","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2f249f4d3ba76d821dea6d18afbf59aea794293bcb88d6ee8ed78f3ea7714fe9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-4-6_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"88463472ba6da2c04df6df5156dfed5441a09f74efd6bddf4e43c096e3295fc0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.07,"score_display":"66.1","source_stated_rank":40},"run_fingerprint":"28d87aba3fe87ded67827c08c2d95038da76e385384747d7879173db99611dcc","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_02f18483087dad439956139b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":144,"at":"2026-05-14","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-14","status":"stale"},"measurement_id":"ab3ab7a82bcd1810de02ab5eb7870767e0eba0599f02687fe8ab1ba2dea3a876","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"standard_error_points":2.8,"upstream_source_url":""},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Sonnet 4.6","source_effort":null,"source_model_version":"claude-sonnet-4-6","source_row_date":"2026-05-14","source_scaffold":"Simplai Agent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Simplai Agent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"7fa8283112df46759e0492a56619d729d2e16344b9b4dd53cc62fb4a7ce6b674","result":{"confidence_high":58.888000000000005,"confidence_low":47.912000000000006,"sample_count":89,"score":53.400000000000006,"score_display":"53.4","source_stated_rank":66},"run_fingerprint":"0761ac9911c567f5531197cbd141185a06c9981e79852a719f64a849c18a6019","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8021d39b92c9db0eb3de055d","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":230,"at":"2026-02-17","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17","status":"stale"},"measurement_id":"a8690be940a52c837662f533b113b55d91f1b36edefd938b0d96ff127577f186","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"standard_error_points":2.8300280743,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Sonnet 4.6","source_effort":null,"source_model_version":"claude-sonnet-4-6_unknown","source_row_date":"2026-02-17","source_scaffold":"Simplai Agent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Simplai Agent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"55bdde596300d68de936c571334e947fb02042b222fb161514255f145f2b9935","result":{"confidence_high":58.973821317727996,"confidence_low":47.880111266472,"sample_count":89,"score":53.4269662921,"score_display":"53.4","source_stated_rank":65},"run_fingerprint":"0fe5d5c31e031dd74390fe57d3e85c4c6db826a2c3e71497c99733a33e4e44a6","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8021d39b92c9db0eb3de055d","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b759adeb3dcd5dd0a3bf1b1ca298ff5d42a16b049b47b3aca77237d64281b5dc","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.572392,"date_is_proxy":true,"latency_seconds":704.225,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.649},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"max","leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":false,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-6"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"a54a8cd4a03c51a856d4b9b37a9a87f7763cccfaf31fcde74fd6181af7c06c95","result":{"confidence_high":58.575039999999994,"confidence_low":56.03096,"sample_count":null,"score":57.303,"score_display":"57.3","source_stated_rank":44},"run_fingerprint":"7620414fcb5fdf60f89978d3f826209c7187d5941f657db2d40712a8a020478d","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":332,"at":"2025-11-06T22:29:16Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-06T22:29:16Z","status":"stale"},"measurement_id":"cb1ec6a040bde1d55000862839e018c4e3da5db0e99e48244e924665a6c01092","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.9,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=claude-sonnet-4-6; createdAt=2025-11-06T22:29:16Z","source_row_created_at":"2025-11-06T22:29:16Z","task_pass_coverage_threshold_percent":75},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"claude-sonnet-4-6","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"f95d8d58fd44415a44867286d8a34d175d04ee582849b985adc9a8cf5ca96347","result":{"confidence_high":72.4,"confidence_low":66.6,"sample_count":1000,"score":69.5,"score_display":"69.5","source_stated_rank":13},"run_fingerprint":"699d59f1c5592df5e7ae6cf8b26011235759f0bbadb9d338c36f1244ff67b274","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=claude-sonnet-4-6; createdAt=2025-11-06T22:29:16Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ea6c6d9a53bd19b358288011","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"31bcbd09a0b42b64548d2815acf867e65b21cd4ecc5e9efd0b00e05e6069f9bf","metric_details":{"expected_trials_per_mode":261,"normalized_gain":20.5,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":13.6},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":56.2,"confidence_low":38.2,"sample_count":87,"score":47.2,"score_display":"47.2","source_stated_rank":15},"run_fingerprint":"209061e382ac90e2297519b0e9919b8330faf6c64a91d83a31713a0db3e20f4b","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f443077f18b018dfdf0194f2","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"4036bc074fb24293a8f2475d35ebe87e11f2a28e7044497eb0292d1b212c62f7","metric_details":{"expected_trials_per_mode":261,"normalized_gain":20.5,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":13.6},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":41.5,"confidence_low":25.5,"sample_count":87,"score":33.5,"score_display":"33.5","source_stated_rank":10},"run_fingerprint":"b422c551aa0dd412f39367656a0d0eb0865e11911ff4cfaccb664917c38275f9","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f443077f18b018dfdf0194f2","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0e5d4e2bf7c898eadb20dd5fcecd75bbc0ab4e07c17cf4081fcd829a9027ebc2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.5760000000000001","mean_standard_error":"4.9","model_release_date":"2026-02-17","notes":null,"standard_error_points":550.0,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Sonnet 4.6","source_effort":null,"source_model_version":"claude-sonnet-4-6_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"e9e15dd1d742e1d47089f11218d797f98d07bfb1d6d4e0b465a88e76a81b33f6","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":43.0,"score_display":"43.0","source_stated_rank":31},"run_fingerprint":"729ee5e92d425e7f25d26c19f92779a706398a882113ae1e17cdf6cd642554cb","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_d4a7e98301a45fe263315cae","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"appworld","metric":"task_goal_completion","name":"AppWorld","release_id":"release_3fa145932216c0afcdd9dae6","split_or_window":"test-challenge-all","unit":"percent","version":"official-c1f56015cf7c"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":137,"at":"2026-05-21","basis":"evaluation_at","evaluated_at":"2026-05-21","first_observed_at":"2026-09-02T21:28:04Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"97c558a666620ee86562bba0f0c3f218f73282a75af6151e219cb8ba46bf6cfb","metric_details":{"interactions":1.0,"normal_task_goal_completion":96.4,"scenario_goal_completion":97.8},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"leaderboard_row_id":"8704e2bf-b2db4444","method":{"name":"IBM-HCL-GP","tooltip":"Hierarchical Component Learning for Generalized Policies"},"repository_revision":"c1f56015cf7c3441ff1933f5bfbec879798d7bbe","source_model_label":{"name":"Claude Sonnet 4.6","tooltip":""},"submission_url":"https://arxiv.org/abs/2605.06957v1"},"run_count":null,"scaffold":"IBM-HCL-GP","tools_allowed":"AppWorld APIs"},"protocol_fingerprint":"fffd4ca8543c5af8b9de05f6bc02595ac6a01c285f093c90520fc13270ab41c6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":98.3,"score_display":"98.3","source_stated_rank":null},"run_fingerprint":"6336f94d4b9c028c2bcb250b8e0efa3899b18f7da3d69c822fc82f50f5e76276","source":{"content_hash":"d44080bb01254150fab30f845d22262a28ee7c90f7850b98ce0ab235ea043f0f","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-02T21:28:04Z","homepage_url":"https://appworld.dev/leaderboard","id":"appworld","last_fetched_at":"2026-10-05T12:33:29Z","license":"Aggregate server-evaluated scores redistributable with attribution (2026-09-02)","locator":null,"name":"AppWorld official leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/StonyBrookNLP/appworld-leaderboard/c1f56015cf7c3441ff1933f5bfbec879798d7bbe/experiments/outputs/_leaderboard.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c0a0659ba50bddf183888efa","provider_config":{"source_id":"appworld","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1ea8664612be99162f0afe7b57bb2aa265e176a2e5dacf63d75a479ba86c0be7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6","source_effort":null,"source_model_version":"claude-sonnet-4-6_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"592565df30e1bc460e6d17b003278a36e3f11fe81435287d81c0fdf1680364be","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":7204.139999999999,"score_display":"7204.14","source_stated_rank":13},"run_fingerprint":"00a8c054041a0b984b29b9fb896a01bf7cdbff67779635e63cc506f1a39b71a5","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2d36a21873281878c5a3aed5","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"104a88c521f8c0981bcb8ce94bf52cb4711c2b6f4245a599047fe48a4ad03166","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-02-17","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort) · gdpval"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-sonnet-4-6-adaptive","upstream_model_label":"Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"fdf67cc0153791509d243b2c1700317efcbc71dfdb249309c28551cb42798b8f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1219.72,"score_display":"1220","source_stated_rank":null},"run_fingerprint":"3a810ec5154564784791791da4278eff64ab0230cc228ad2c45f671dd693fd27","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a8b0f60521f1c0e2d9872735f5f2f53d7630f0ca9d0bf31fee5be551307ea4c8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6 (max)","source_effort":"max","source_model_version":"claude-sonnet-4-6_max","source_row_date":null,"source_scaffold":"standard","upstream_leaderboard_url":null},"run_count":null,"scaffold":"standard","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"420f7d7c5f6431f39a1c12e26e98cb549477c9ac8537259a07c290e052f5f225","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":8.3,"score_display":"8.3","source_stated_rank":13},"run_fingerprint":"418f845b87b966cd28f0d73d4587d1ce2b8918b5b35e7909d9c7de53a4380105","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_258fcb8ee8e5667bc1ac3d85","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"384fde758b85b6d084138177589af78991a049bf1ffaa3d7dcb7f70a561c4498","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"anthropic/claude-sonnet-4-6","name":"Claude Sonnet 4.6","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.6 (medium)","source_effort":"medium","source_model_version":"claude-sonnet-4-6_medium","source_row_date":null,"source_scaffold":"standard","upstream_leaderboard_url":null},"run_count":null,"scaffold":"standard","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"dbb23579bb8eaca30a3b0da07412e239e56f21d5e640a8ec0e03cd080921d7d9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":9.3,"score_display":"9.3","source_stated_rank":12},"run_fingerprint":"482dacdc20b8e19bad774540133f5d2482f366f8ec659695a641ff3f899888b1","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_02f18483087dad439956139b","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6d4755d0e94438d8a68d4710f1e7fcec7a3b75f61e8918b50e493f3ebe3fe7d2","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-08"},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":false,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Muse Spark; model release: 2026-04-08","source_accessibility":"API access","source_model_name":"Muse Spark","source_model_release_date":"2026-04-08","source_model_versions":["muse-spark"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"82d4883296c9b2f33f8bb1932edf479540a26521eb98ef2f43ef6fdbe499edf2","result":{"confidence_high":155.11,"confidence_low":149.77,"sample_count":null,"score":152.09,"score_display":"152.09","source_stated_rank":41},"run_fingerprint":"53eafe4837a71071de3387ad877bbe4254c456046baeca43483d873a387f1efc","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8fb9ab5df9eef640038582d9","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"83a6eaeb3bc10c9abe9a2d564f21ecbf0655ef547953ee6722780a591536be74","metric_details":{"license":"Proprietary","variance":8.588605865852447},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1495.060634033421,"confidence_low":1483.5727663877967,"sample_count":14120,"score":1489.3167002106088,"score_display":"1489","source_stated_rank":15},"run_fingerprint":"edd24a1134656d2114dc2163a9668864e9878b149ce485085a6033a93d26ba79","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54436950921b18ec0c2e019b","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":179,"at":"2026-04-08T16:57:23Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-08T16:57:23Z","status":"stale"},"measurement_id":"6f1fba9acf6244b0560858d05358d1721319c2503d3e5c369bf0d3290f8b5fce","metric_details":{"calibration_error":50.0,"confidence_interval_half_width":1.92,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=Muse Spark; createdAt=2026-04-08T16:57:23Z","source_row_created_at":"2026-04-08T16:57:23Z"},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"Muse Spark","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"dc87d816174c4eb17ed79e0d026077ade8ce4b27042a684752d086c21213e157","result":{"confidence_high":42.480000000000004,"confidence_low":38.64,"sample_count":2500,"score":40.56,"score_display":"40.56","source_stated_rank":4},"run_fingerprint":"4957fbdceec8d7d8f1f03406460d605a0177abc7a23bb7675861b7e15fda5b28","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=Muse Spark; createdAt=2026-04-08T16:57:23Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_353e71505b6265ea76a3923f","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"38f12535928983bbf27242b8a068e4acbe328694e631e2e0f135717ccc591a56","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.003074,"date_is_proxy":true,"latency_seconds":94.58,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.546},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":88.384,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":131072,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark","zero_shot_accuracy":90.909},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"a5db52abf58df8be2fd679a883baeae43c995bdd3b2c1c37e270fb7939de1568","result":{"confidence_high":92.67616,"confidence_low":86.61584,"sample_count":396,"score":89.646,"score_display":"89.65","source_stated_rank":28},"run_fingerprint":"e4ab39591b8dc4257f6a08894a8c685df5119ae424b900a29e9a61b8e9e19d3f","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7503d5cd94f11a21bb05a963","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":179,"at":"2026-04-08T16:29:51.099Z","basis":"evaluation_started_at","evaluated_at":"2026-04-08T16:29:51.099Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e1b1d08db565fafb9ff0a7b85c9ac93afdf0b592d715555d88541c591ea207d8","metric_details":{"best_score_across_scorers":"0.898","model_release_date":"2026-04-08","stderr":2.1999999999999997},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"muse-spark-pre-release-gpqa","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"0e598ca0d590f6f4e6a6b1f4fc1dfa30e7b06b7f47b013dc5d417231ea428993","result":{"confidence_high":94.112,"confidence_low":85.488,"sample_count":null,"score":89.8,"score_display":"89.8","source_stated_rank":49},"run_fingerprint":"1ad8140809a072908f77e4aba1e2bb8fb4bcf0fbebee745ab8a4d69fe6106a96","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bc74738cfea7c282dccf2598","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b99e82061a8e21020f3f769a39306dfc5028f392662a526735a947d280951379","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.00167,"date_is_proxy":true,"latency_seconds":46.242,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.327},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":131072,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"bd3e750777eb10be208fc9b70d37ffdbc90e74223823b35fadf2e1fcdc1deafe","result":{"confidence_high":87.95692,"confidence_low":86.67508000000001,"sample_count":null,"score":87.316,"score_display":"87.3","source_stated_rank":30},"run_fingerprint":"c8deb0e6065cde32d127ec89d1b7005edf779ce328cdfd0cfdf1b57e31e55d95","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1f4805d656bfdb6334fd4c90","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":179,"at":"2026-04-08T16:30:04.184Z","basis":"evaluation_started_at","evaluated_at":"2026-04-08T16:30:04.184Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c65c5f165e8cca0a9801467598321eca3a9ea53a09dc535c109ee646fe12ce57","metric_details":{"best_score_across_scorers":"0.889","model_release_date":"2026-04-08","stderr":4.7},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"muse-spark-pre-release-mock-aime","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"828ff4b76440946102b6be1ecff774931e036c25110ae2a236ffab6b1605b468","result":{"confidence_high":98.11200000000001,"confidence_low":79.688,"sample_count":45,"score":88.9,"score_display":"88.9","source_stated_rank":70},"run_fingerprint":"2a9cb8ceecaeb8fafcdb1cf04412240e761762841f50735378bb8193f0a3debb","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_34bc8136627a1b5e0353c16c","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d66017294a73905ddd0962689d1ce368546b9b1d92b6a1236bab0b39e3ef7b59","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.001746,"date_is_proxy":true,"latency_seconds":48.272,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.798},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":131072,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c70493aeb7190161af8bdf4a0c253c647050223d1ea59bcedf6034561741b6b1","result":{"confidence_high":88.96308,"confidence_low":85.83492,"sample_count":null,"score":87.399,"score_display":"87.4","source_stated_rank":16},"run_fingerprint":"d8a4cb5ae33c1d271661cce10d396cd4389e8dd1fc9daa5f401ff5df97f8621c","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8166611feaa70b222ae62c85","provider_config":{"source_id":"vals-mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ee67e28929da89fe49a5550e2fc1523d505e15158114ee3eb16fd6c044aeb2aa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-08","notes":null,"upstream_source_url":null},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark","source_effort":null,"source_model_version":"muse-spark","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"65600d09ac84b2d20072d49a794cfb11bf60e2bbae25e4f9cc8ef0e82c365666","result":{"confidence_high":20.765983,"confidence_low":5.865021,"sample_count":71,"score":11.330612244898001,"score_display":"11.3","source_stated_rank":89},"run_fingerprint":"8ab9eccf859583954d81a0d3950701ad8318eeeb89daa872c65d3eec26793a6f","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5638767fc11cbaf7aa5af326","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ffbdf56492982ec7fefe567b03aa70a10b1b26964e50aef1e1cfeaca00196c11","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.444473,"date_is_proxy":true,"latency_seconds":495.294,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.954},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":131072,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"5a65a094032c52078c83a76cc60944f43c7e6365e18a34c11351fe7ca6434445","result":{"confidence_high":78.22984000000001,"confidence_low":70.57016,"sample_count":500,"score":74.4,"score_display":"74.4","source_stated_rank":48},"run_fingerprint":"ffe840b3cce2edc12cd18ef5df72763f6550db6f903d4b0429d16d5ab7e98296","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c2ccf01d838a7e46f5423db1","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":179,"at":"2026-04-08T17:04:49Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-08T17:04:49Z","status":"stale"},"measurement_id":"97064bf34d347564e4604462abaacdc67cea1fa57b9fe9c54c608078f43554a1","metric_details":{"confidence_interval_half_width":3.6,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=Muse Spark*; createdAt=2026-04-08T17:04:49Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2026-04-08T17:04:49Z"},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":true,"cost_cap":"uncapped","protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"Muse Spark*","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576","turn_limit":250},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"2e8ba66e5c8d5dab5c1495a2699ba661fdff0af90fd1d266edb11cabebaec772","result":{"confidence_high":58.6,"confidence_low":51.4,"sample_count":731,"score":55.0,"score_display":"55.0","source_stated_rank":3},"run_fingerprint":"3cd7dd2b7b785915ca576c506ade49e0eec4ae90af8a314a5a0fb81ce60a6869","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=Muse Spark*; createdAt=2026-04-08T17:04:49Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_832b2d179e92c731edf99d2c","provider_config":{"source_id":"scale-swe-bench-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0018d2822be717d44cc7137365d1aa1dab1bc1b41accb6f5fdf395472f5d7529","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-08","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Muse Spark","source_effort":null,"source_model_version":"muse-spark","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ff35bb91ca728f3e1a6604a6494745a4224ae935d13a0e1e1741a70135b63771","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.5046296296296,"score_display":"51.5","source_stated_rank":79},"run_fingerprint":"1d7dbd88c79b266ddeb13dcc701ee732b2311381cc1b3aaff48112db0196207f","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7c87f12cd9a3ae0363f7226b","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7238f5a076e481108f90c57f27b6f7a85eb7564d20404bd146125ff6b6bb6aa1","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.579705,"date_is_proxy":true,"latency_seconds":981.298,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.966},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":131072,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"meta/muse_spark"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"68269a494457bea25345f241295dff350e41edbf6c6c800b535353c4ddf8690a","result":{"confidence_high":27.44736,"confidence_low":11.90064,"sample_count":null,"score":19.674,"score_display":"19.7","source_stated_rank":77},"run_fingerprint":"c993484b475f365653857bf53ce61d7114f189510eb4ba239d95729b5d70d38a","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_caf27ae6f0e02b627cb46007","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4498941768a8a283f83b272bb7ac185aa329841aab6b2cf71a7a45d9399c7a94","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-08","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"muse-spark","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"7478f140435c0398c7174a5112f26ce28b9af101e641529581b4f498d38610fe","result":{"confidence_high":1528.6,"confidence_low":1496.47,"sample_count":1608,"score":1512.53,"score_display":"1512.53","source_stated_rank":47},"run_fingerprint":"943888c88cc9423da178823af254d1b5d89f25b2acd7a491e8fbcbc20a73177e","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3bf852297bfa61c449d25840","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":179,"at":"2026-04-08T16:48:43Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-08T16:48:43Z","status":"stale"},"measurement_id":"ed9dfa3036b70adb58d116bc3139b21aff4f45ff08718395beabec78f5ecd111","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.3,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=Muse Spark; createdAt=2026-04-08T16:48:43Z","source_row_created_at":"2026-04-08T16:48:43Z","task_pass_coverage_threshold_percent":75},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"Muse Spark","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"962ba9b1a56e73d6dc062c910f0e135365127a94d6b23df078322fa70da25295","result":{"confidence_high":84.5,"confidence_low":79.9,"sample_count":1000,"score":82.2,"score_display":"82.2","source_stated_rank":2},"run_fingerprint":"28e9b328dbdbb4e5408715d30eaaa3f73a41bb12901783534a413d527f2842b2","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=Muse Spark; createdAt=2026-04-08T16:48:43Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c20d16cecbbd58186fa001f5","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d0505bd35aa7cb214b2355276b292c40a4c188b8ddf615843c591c0723978f93","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-08","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Muse Spark · gdpval"},"model":{"id":"meta/muse-spark","name":"Muse Spark","provider":"Meta","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"muse-spark","upstream_model_label":"Muse Spark"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"86a85894bc28ebaba58b3af7d7f256d8dea8cca5d7d40386f008514ff27d9d32","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":985.78,"score_display":"986","source_stated_rank":null},"run_fingerprint":"a170548c9c48606abd31b42010558eccf228baa62f883f9702702553d7187337","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Muse Spark · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d20f8a5e90eacbc5f8ebdd68","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e39b7c66cd92b0335c364c67e83afb60d8bffbbf5d7698a2b492d5d8053216d3","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-17"},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Grok 4.20; model release: 2026-02-17","source_accessibility":"API access","source_model_name":"Grok 4.20","source_model_release_date":"2026-02-17","source_model_versions":["grok-4-20","grok-4.20-0309-reasoning"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d734b7b05a04349756eb173fc6442ea28ce46896e399f627cccb3b83d384e466","result":{"confidence_high":154.38,"confidence_low":149.32,"sample_count":null,"score":152.02,"score_display":"152.02","source_stated_rank":42},"run_fingerprint":"27e89c3775a7e1014128bd6eb64ab2d54decbf9bf12c45ca70fd2a3543cfdf25","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_907345bd817b4bd0177c506c","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"74348a3a10992face1f5a1145370374059a0225bc605391ece2be247541ca5dc","metric_details":{"license":"Proprietary","variance":3.5244003173040577},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1475.2048963507377,"confidence_low":1467.8458641625857,"sample_count":66334,"score":1471.5253802566617,"score_display":"1472","source_stated_rank":44},"run_fingerprint":"5b41d73bb511661466e7bf506778bf3a4598ef61131970f0882af22140890665","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b07daaa1bd4a52b03c8c8e15b837867f73469d3246d57f940f9804329155637b","metric_details":{"license":"Proprietary","variance":5.455300036316514},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1479.1915885016276,"confidence_low":1470.0359759182904,"sample_count":27827,"score":1474.613782209959,"score_display":"1475","source_stated_rank":36},"run_fingerprint":"0440726e67d45150757d301833236c758140f982cef49a5da71960a478849393","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_306bf5ad4252fa8f18ab95cf","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:29:44.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:29:44.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"077a7e4e61e95f7fef36f4ee127eb658dab04b8fe0591e61e8612228e449365b","metric_details":{"best_score_across_scorers":"0.302","model_release_date":"2026-02-17","stderr":1.4526080235459493},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"GVm7khGvtdnQKGfkgAHD8q","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGVm7khGvtdnQKGfkgAHD8q.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/GVm7khGvtdnQKGfkgAHD8q.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"4605760acc55f59e9cd7de869182ff241c553644767751f7e85e6c82330175d4","result":{"confidence_high":33.04711172615006,"confidence_low":27.35288827384994,"sample_count":1000,"score":30.2,"score_display":"30.2","source_stated_rank":66},"run_fingerprint":"3add1ffe802e329fb622559c00e6b13a1d8b94028c4cf3c2df6e708e262607ab","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":206,"at":"2026-03-13","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-13","status":"stale"},"measurement_id":"835a2b56cacaec1187c4d88734b6acb56750bc6a78656432620aea856a1c3321","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run; vendor scores using a different agent or tool setup remain separate supplemental runs.","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","cost_per_test_usd":0.034461,"date_is_proxy":false,"latency_seconds":59.421,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/grok_grok-4.20-0309-reasoning","source_locator":"Results · Overall dual-prompt accuracy","stderr":1.594},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":87.374,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":2000000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9dda0ab1137e45a9d7e9c1db2d162a1b5ca7ea99a72618fac5e3a8ff547e22aa","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.20-0309-reasoning","zero_shot_accuracy":89.899},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"94f944c6c7b8e090f3733521974b5291b0e850823b8fd44b897422fbad810867","result":{"confidence_high":91.76024,"confidence_low":85.51176,"sample_count":396,"score":88.636,"score_display":"88.64","source_stated_rank":34},"run_fingerprint":"d6d1b73ddadbe0748a79e40fc14210697ee7ded6576a0be9453e7b9c7910dc51","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":83,"at":"2026-07-13T16:45:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T16:45:48.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ea5d75b4efe1655f06e7625cfc0a7fcfc252355729e1cba36cc29c6ea280d311","metric_details":{"best_score_across_scorers":"0.8933080808080808","model_release_date":"2026-02-17","stderr":1.8633389514966319},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"STxFZiSPAPScUuENvXBWLc","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d2b38b90ba87cba0d068a9fab923ed6e26ab59dfb895cf878e248c4cdb4f5101","result":{"confidence_high":92.98295242574149,"confidence_low":85.67866373587468,"sample_count":null,"score":89.33080808080808,"score_display":"89.3","source_stated_rank":52},"run_fingerprint":"439278012115857e77ffc1b32e3f6fcaa2a51cc74e273db0f7b2f6bbf56b238f","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3d0f2cd2287422a5c1912b25774ecea62ad76932e39ded6bd4b4eb72501f504a","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.012836,"date_is_proxy":true,"latency_seconds":11.654,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.34},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":2000000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.20-0309-reasoning"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7ce46faec7b43be71a1e96fddcd35d9d03bc9d967869a8d85c0abe4a49d70a46","result":{"confidence_high":86.9204,"confidence_low":85.58760000000001,"sample_count":null,"score":86.254,"score_display":"86.3","source_stated_rank":44},"run_fingerprint":"3599b47dd4ad40c76920af0fbcfd8aca70638b971fb9938da3d1d2aded167338","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T16:45:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T16:45:48.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b26ec843a872cef1f4814e7b342950cec1012adfd7be6f941acc339aad0993f6","metric_details":{"best_score_across_scorers":"0.9222222222222223","model_release_date":"2026-02-17","stderr":3.2829739672413996},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mM6vdJpgjLY9prnetXzZJS","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"82e13eaddc2ac54e19768a7bd6791e3aa6e226b89a4a8d14629617a9d47cd1a4","result":{"confidence_high":98.65685119801537,"confidence_low":85.78759324642908,"sample_count":45,"score":92.22222222222223,"score_display":"92.2","source_stated_rank":61},"run_fingerprint":"b6f269270ef9278fb937271ff4c0e9510385d0a1f808530220c1d09fee1a02f4","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T15:51:35.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T15:51:35.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b305bbdfbaabebb3f9c11ad81072ba7d27e0c056e1dff80810d2a7b3ee7055dc","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.44912280701754387","leaderboard_private_problem_count":285,"model_release_date":"2026-02-17","public_example_count":10,"stderr":2.9515543245521676},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"FLDt53waLTeshC7PK9uVy2","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFLDt53waLTeshC7PK9uVy2.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/FLDt53waLTeshC7PK9uVy2.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"","thinking":null,"upstream_model_id":"grok-4.20-0309-reasoning","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"107b35476888ddc67d09c05c0c48dd53aee5142afadbc44eb185acc3f8cec81d","result":{"confidence_high":50.69732717787663,"confidence_low":39.12723422563214,"sample_count":285,"score":44.91228070175438,"score_display":"44.9","source_stated_rank":52},"run_fingerprint":"320861e3fa35774f66e4dae0dbf626a4acb2ed1b4acfc9a0ee445b26f1e24b25","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"62071058ed5080d6a6981f76da530ba816a7a27f14766c23b9ac2b3041c48e25","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.92,"model_release_date":"2026-03-09T00:00:00.000Z","results_url":""},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"grok-4.20-beta-0309b-reasoning","model_type":"CoT","upstream_model_id":"grok-4.20-beta-0309b-reasoning"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c3fa0592b6441af3735d852e2d062d733783e4f6e00f6d83561e492ebf372c72","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.14,"score_display":"65","source_stated_rank":66},"run_fingerprint":"da3666a3a21596ec6bd265bd3a97d2f1d95dfd0858e074ef069c24fcf6e2a556","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1568f5830ca8ac526a1a8d7aa9c43626abf249f7f929c6562eadde5a0deeaee4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.92","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.20 (Reasoning)","source_effort":null,"source_model_version":"grok-4-20","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e24ebb5dc505b74f968f007ab1874ca7ac3aec69a9a15f3f806f0c666bcf1db4","result":{"confidence_high":69.879273,"confidence_low":60.081017,"sample_count":360,"score":65.14,"score_display":"65.1","source_stated_rank":69},"run_fingerprint":"6edf6c55551cf4ea3284a560e5ada91cc47d2928b31f95a579ebdad537fcbf83","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f1662580b7523f8dd05a47c4","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T15:51:35.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T15:51:35.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bbfe62728f384679bbc6ea9e03e13b3e5cc1761f2091ac69285e93b27d54a337","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.17073170731707318","leaderboard_private_problem_count":41,"model_release_date":"2026-02-17","public_example_count":2,"stderr":5.949419959829496},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"SX5FCWMk3HF83gGsP3WFA6","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSX5FCWMk3HF83gGsP3WFA6.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/SX5FCWMk3HF83gGsP3WFA6.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"","thinking":null,"upstream_model_id":"grok-4.20-0309-reasoning","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"55c3e2eabbee1ba0c01a0bdac6bfb587021b6787918995cf0ae59abb2103760a","result":{"confidence_high":28.734033852973127,"confidence_low":5.412307610441507,"sample_count":41,"score":17.073170731707318,"score_display":"17.1","source_stated_rank":48},"run_fingerprint":"a7222ca6a7d048be1067f81af2110b2ec1307c22394df6f9e2f7406bb61c767d","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":206,"at":"2026-03-13","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-03-13","status":"stale"},"measurement_id":"a7a2adc5b6f6fe0bf8aaf8ac309974c543b7181c128dc3ffb2fdf1d802631b11","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.026669,"date_is_proxy":false,"latency_seconds":44.461,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/grok_grok-4.20-0309-reasoning","source_locator":"Results · Overall accuracy","stderr":0.893},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":2000000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ef80d34c2c94f614c1bbad9d4cbbae5623958385f53120658d9669995074f600","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.20-0309-reasoning"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"73402615af0402aaa2ff411f7757e3539429be6521d5a76511a02e38a8cffc04","result":{"confidence_high":85.21828000000001,"confidence_low":81.71772,"sample_count":null,"score":83.468,"score_display":"83.5","source_stated_rank":32},"run_fingerprint":"519f9fe4d9c030ad98481dba3bb270995593587e4002a627e3474b8eb52e7fe9","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":83,"at":"2026-07-13T16:45:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-13T16:45:48.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4f6de14d3a1bed5a2e5ec93b9eca4f94b932d2163f03e63345b8062befd2ef2f","metric_details":{"best_score_across_scorers":"0.24","model_release_date":"2026-02-17","stderr":4.2923469599092785},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"DMei2tkTbCqecGNjYCYYoW","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b9e28cfd22ec6c1e5d25549f6253b0c85ad47eea478bd25c7a74887ae3b1139f","result":{"confidence_high":32.41300004142219,"confidence_low":15.586999958577815,"sample_count":100,"score":24.0,"score_display":"24.0","source_stated_rank":69},"run_fingerprint":"d1f6da0faf4a4183ecb9888e4a24695d3779935c3c1d2212fc51939f8aaba8ff","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9633b46d7111052a247fd0e83e85dfc86017f577809878e7b3296593cd660bb0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.35","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.20 (Reasoning)","source_effort":null,"source_model_version":"grok-4-20","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c6afb855bbe79a876e8648668d61283fd4fa5aec6eb157ac1907db4cb7ba6129","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":89.5,"score_display":"89.5","source_stated_rank":77},"run_fingerprint":"c7fcd7a2ba9eb90d2ee4638da7877a770544e03c8610e39f6344fc16e4b883d9","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_899434b1ccae0f0aa50272f9","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef287e0c2c342a40cdc10cca97fd4e96a18dbdf60a4ed7976b310c107c5eed72","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.515946,"date_is_proxy":true,"latency_seconds":134.759,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.006},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":2000000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.20-0309-reasoning"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"aca6cd47ded92a07450645b63f110119496da936ecbf0c61d9ff69be7a8b523d","result":{"confidence_high":76.13176,"confidence_low":68.26824,"sample_count":500,"score":72.2,"score_display":"72.2","source_stated_rank":56},"run_fingerprint":"8570abebb877c280cb843edb7e590751a7a6da83e2e6b7605cbdd7ebeb1eb47b","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"90229518bfc1eca72840a58a692eedb87960ca6dda86934516e237d73c43504c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.05249,"date_is_proxy":true,"latency_seconds":76.09,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.028},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":2000000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.20-0309-reasoning"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"286e6257c072267498f5c5011de7305e42b8e8557f12d47247b9a48a6951b4de","result":{"confidence_high":86.27988,"confidence_low":82.25012,"sample_count":null,"score":84.265,"score_display":"84.3","source_stated_rank":40},"run_fingerprint":"136c82456a503fecd847e5aafc63036492d48a169d69d0a16c1a2afc8c10261c","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fdcbbb3cde9bc0fcf1d971fbaff64203bbd8e923c8f9b488e9cf5bb4df33f55e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.13838","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"grok-4.20-beta","source_effort":null,"source_model_version":"grok-4-20","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"8b0eb079783f5c9bb32287042ce567a1452c3399a3ddbc9506476318f495e6ac","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1150.28,"score_display":"1150.28","source_stated_rank":41},"run_fingerprint":"3854adadcf3aef7e17310ec18cc5f419bc510fd857fd5ebd6cb37bb8aa50820c","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c7ba76e47c689d327667a5a8","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e2db929e1b8df906144856932402414fc2443c7e0fb28be80ac581c6f1f79cd9","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.771885,"date_is_proxy":true,"latency_seconds":301.651,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.058},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":2000000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.20-0309-reasoning"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"4199674677e17f7f38bb7c173b01d6f92e0967ec808fd6a704f9f589d5cf8411","result":{"confidence_high":8.09668,"confidence_low":0.029320000000000235,"sample_count":null,"score":4.063,"score_display":"4.1","source_stated_rank":96},"run_fingerprint":"61221a6aca346ed2a878b23fe0baa35438efb86edf61db85351810226668cff0","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef89c2bd61490c269007e6b1606b7991b76a03eb15c34bfc3036d4a4cff4fb15","metric_details":{"license":"Proprietary","variance":9.53920639760997},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"grok-4.20-beta-0309-reasoning","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"76d925344c565b601af95c3fce4d33e0ec43bd02642e41ade087edbfc8ee2ba1","result":{"confidence_high":1380.130650573104,"confidence_low":1368.0237156086268,"sample_count":15503,"score":1374.0771830908652,"score_display":"1374","source_stated_rank":97},"run_fingerprint":"7b3beb8ece83ef32d124ae4f80fbfd2f9063140f95da79744c1c2479de5e6eec","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"739f53606982cc4654bbc884866bc94fa719377ca06d414342adb0b98c107c09","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4-20","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"353c093620e2d7d64f66a4c57e342cb73938049d62b01ceca234305899d66b44","result":{"confidence_high":1379.98,"confidence_low":1367.53,"sample_count":14571,"score":1373.76,"score_display":"1373.76","source_stated_rank":96},"run_fingerprint":"57fe8b1ad74cdedc07160f6c691eaeaecbc26ca5546b0e52102d38f90047249f","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dd6dcf53b801c83702c09350","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7c73513b48c5de101ff7a45ed9dd0ac8081b0f915fc387e7285a5487a7b2fa4d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4-20","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"fba795302480298a60aef10f1c2059807bfae19ad21f591caad5eeb0d0cf6c22","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.26,"score_display":"52.3","source_stated_rank":67},"run_fingerprint":"87eb492020225be681a6e705ffc82f62f533f1bafa5d74b512b9c16e43ad2d5c","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_97752946e06fcd42f05804fd","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":186,"at":"2026-04-02","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-02","status":"stale"},"measurement_id":"b1f0a59f540700514e59130bfa3fad6583c7c96672b90032ee0f8fb9e4862b3e","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Grok 4.20 Reasoning","source_effort":null,"source_model_version":"grok-4-20","source_row_date":"2026-04-02","source_scaffold":"grok-cli","upstream_leaderboard_url":null},"run_count":null,"scaffold":"grok-cli","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"bf12f069671c64bc269048a0f239ef52bd175692c6e1793f1bc1813746466367","result":{"confidence_high":null,"confidence_low":null,"sample_count":89,"score":57.3,"score_display":"57.3","source_stated_rank":59},"run_fingerprint":"c8bd6a589a466e7e93c330fa23a09292c1783751b7efdc30de6cd5a2d543e981","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1271c453e8973d44b75df888","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":209,"at":"2026-03-10","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-10","status":"stale"},"measurement_id":"8a86c2cd6319e5998fedcbfd44881454eecd7d171861fa02e5331ae863003ef0","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Grok 4.20 Reasoning","source_effort":null,"source_model_version":"grok-4-20","source_row_date":"2026-03-10","source_scaffold":"Grok CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Grok CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"22776b33a3fce8c8261e7807e7cd0c486d39b0d54c5aec33bd0c846fa9e945b8","result":{"confidence_high":null,"confidence_low":null,"sample_count":89,"score":57.3033707865,"score_display":"57.3","source_stated_rank":58},"run_fingerprint":"10a49b487a731c506af694bcfe186f9f3800b3c0a268ab8dec173c76309a911a","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1271c453e8973d44b75df888","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cf25d8eaa79c7c3aaa5983865ee09ac615645f231f5e0fd14bc7e7ef1c707ef6","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.407362,"date_is_proxy":true,"latency_seconds":305.726,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.991},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":2000000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.20-0309-reasoning"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"5b9975bbd279d7e644fdbf0e2541ed95299a96e80987cbcb551be3f823579a3e","result":{"confidence_high":46.13736,"confidence_low":42.25264,"sample_count":null,"score":44.195,"score_display":"44.2","source_stated_rank":65},"run_fingerprint":"eacb40b93a23163e800ebe7cdfab92c9cfb2c6385a988ab00d760356864fdc16","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_586ef081fe8e74ae64d06ed8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"reasoning","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"82fd6c4d1e7c2c8c8c616591b79f98dc183f041a0cbd04cfcf09ee08bd63434b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-17","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-20","name":"Grok 4.20","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.20","source_effort":null,"source_model_version":"grok-4-20","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"9c035e4c8ddd029fc4d331442cb9952ff1acef4af6cb436912b4d82ed4942647","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4662.846666666667,"score_display":"4662.85","source_stated_rank":32},"run_fingerprint":"28622093b0d8aff6f5c115f2cb60ab9ba9777b8be67316781dbd42cace9f9e9c","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_91997adfb8f527350e6f7228","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"88fd4096ec9fddbffdbf0d7a8626b80186a31eecd0d728e183c49d87a93e4a78","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-08-20"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GLM-5.3-Flash; model release: 2026-08-20","source_accessibility":"Open weights (unrestricted)","source_model_name":"GLM-5.3-Flash","source_model_release_date":"2026-08-20","source_model_versions":["glm-5.3-flash_max","glm-5.3-flash_unknown"],"source_reasoning_efforts":["max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"42800920f0c4ffa9681fc842f855482f0d023b732e139401dc1cdf4c933af574","result":{"confidence_high":154.28,"confidence_low":149.38,"sample_count":null,"score":151.94,"score_display":"151.94","source_stated_rank":43},"run_fingerprint":"4ceb996557da392e245f4e6f9d81a4ee1cbfb86b4b7c0ef20196a125e2d06c52","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4d4e599ec811239ce079646f","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d229d36c281e0bbebfb114b91bc918e3685fefb7d364a1eeb15e0cec9310a0f1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · GLM 5.3 Flash · intelligenceIndex"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-flash","upstream_model_label":"GLM 5.3 Flash"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"6107dbb63b2a0c5d45b2f2547bc095754f24c940831297f726978f0f348531f6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.807466,"score_display":"41.8","source_stated_rank":null},"run_fingerprint":"82613eced780f35800afdad8782dcac15a8b71b348389fd5a0c2e4c41dfeb9c5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM 5.3 Flash · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_626f9b1c4e05a66a41042aa2","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"87335db461c3143dd9ccab444c03057bc95ae7df4790620ffe5c97a2b407b942","metric_details":{"license":"MIT","variance":7.4305131661600505},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1478.3431494987472,"confidence_low":1467.657830534031,"sample_count":22971,"score":1473.0004900163892,"score_display":"1473","source_stated_rank":41},"run_fingerprint":"29a262d18b5572f6f65a7193a7bf73cd1b2cba3f4f57c4e96bc1f0ff24679789","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_939a9cca21255a0df9b67f4d","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8ac60e43f16ad9c2d62a944a3271b3ab7cf47211ae519503d3fa4cb85e8c1715","metric_details":{"category_scores":{"Agentic Coding":56.76766666666666,"Coding":78.95,"Data Analysis":76.401,"IF":52.820750000000004,"Language":77.30733333333333,"Mathematics":81.2445,"Reasoning":77.64425}},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":71.59078571428572,"score_display":"71.59","source_stated_rank":52},"run_fingerprint":"27d4414f061483735ef81f9758f373099aa0d9e62715e1a4b628d437c9144f86","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ea7550e0cb1aa3e5a0a3c455","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":40,"at":"2026-08-26","basis":"evaluation_at","evaluated_at":"2026-08-26","first_observed_at":"2026-08-30T10:48:10Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f52ff8e2e18da86ead79661bd3c6c1f9a33298b72b1846c09070bc8e71416913","metric_details":{"modality_lane":"not reported","notes":"HLE with tools (full set) and a 300K-context management strategy, not the no-tools default; judged by GPT-5.6-luna (medium).","num_parameters":321323031390,"pull_request":null,"result_file_url":"https://huggingface.co/zai-org/GLM-5.3-Flash/resolve/main/.eval_results/GLM-5.3-Flash.yaml","source":{"isExternal":false,"name":"GLM-5.3-Flash model card","url":"https://huggingface.co/zai-org/GLM-5.3-Flash"}},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":"GPT-5","run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"cdd0be69efb364191bf0190a287662dcbb64c930f0f964558a199ae456ef38fd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.3,"score_display":"55.3","source_stated_rank":6},"run_fingerprint":"2ef5f35297807c44138b7310115afb38a852f83b0a18267e4627e5f8b8a1b22f","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_87d39e2d4e9d451f3ef67d65","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"24beccbbfddf3436eea8bb2d302ddb0844fededf4a1be76ec78919bf050a2a1c","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · GLM 5.3 Flash · hle"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-flash","upstream_model_label":"GLM 5.3 Flash"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"153cbf6a7a9f294e95e200195b1d014fa9dbd045789018707c2ea95d7e996a21","result":{"confidence_high":41.933683,"confidence_low":37.805814,"sample_count":2158,"score":39.851715,"score_display":"39.9","source_stated_rank":null},"run_fingerprint":"8a6c3c899602655e41d47292e4cae2b86a0e22bbbdb9991df2e2052a4054c67a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM 5.3 Flash · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_626f9b1c4e05a66a41042aa2","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a6d964765ff63bfe5d22ce91073ac16b7f3c411884cb3b89f51ae39ef1b6d21f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.001626,"date_is_proxy":true,"latency_seconds":131.004,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.126},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":85.354,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"max","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3-flash","zero_shot_accuracy":87.374},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"db11708ac263fa1599180bf31170ff3f7404745cd7fa3e62c9a1cc2e7c2b86cc","result":{"confidence_high":90.53096000000001,"confidence_low":82.19704,"sample_count":396,"score":86.364,"score_display":"86.36","source_stated_rank":43},"run_fingerprint":"b8e16d279ff6d968ee2b7e016b4b5009985c73ca9210c963a7b6f48bca6ae253","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fdfbb1cd161557258da456f100f51e6d81fe61dca05dd463e0483974632691df","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · GLM 5.3 Flash · gpqa"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-flash","upstream_model_label":"GLM 5.3 Flash"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"67babd7f9f883cc00defbdb462f8f0bc251683a779a15332403141cf56a4af9f","result":{"confidence_high":94.411604,"confidence_low":86.443879,"sample_count":198,"score":91.212121,"score_display":"91.2","source_stated_rank":null},"run_fingerprint":"23388153362256eec6fb52f34735efa718d3f9dbb82c821bd0149a2012aebb05","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM 5.3 Flash · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_626f9b1c4e05a66a41042aa2","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":39,"at":"2026-08-26T22:26:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-26T22:26:55.000Z","first_observed_at":"2026-08-28T02:18:20Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"30db3f8dac9ff90c0454f2d988b3774fefa3bc46d0e37b144e8034ca401e7d95","metric_details":{"best_score_across_scorers":"0.9015151515151515","model_release_date":"2026-08-20","stderr":1.7183231147411449},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Jvt5BNr4ERbE9QGgMcEbmT","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6555cf41a585977b7600e9b797d4ba466194f0ef78dfa1e524af55b37a79c9e8","result":{"confidence_high":93.5194284564078,"confidence_low":86.78360184662252,"sample_count":null,"score":90.15151515151516,"score_display":"90.2","source_stated_rank":44},"run_fingerprint":"404c138406502f5a05dd6fce1c9a54e64d15b5079938f46420c36dd9c7b1bf81","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4f37e19cac7730d0291783d4772aa20696b5973440802b23966a860ea77bd9aa","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.001058,"date_is_proxy":true,"latency_seconds":44.762,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.341},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a5d73f87e55dffb33962332713a945020dea7dbb5ed53d640071f0e6f0cb7e51","result":{"confidence_high":86.72736,"confidence_low":85.39063999999999,"sample_count":null,"score":86.059,"score_display":"86.1","source_stated_rank":49},"run_fingerprint":"73d3faf0350e29dde7089f8eb750251ad61016d7fc240e92bcef476cbb119dfd","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":39,"at":"2026-08-26T22:26:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-26T22:26:55.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a432d390ba8f95a0259692c39147320500a2dee7bad1f1cf614014280f7a1509","metric_details":{"best_score_across_scorers":"0.9388888888888889","model_release_date":"2026-08-20","stderr":2.162226738085332},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"NCikvVWc5SEyMEbCqvVNb2","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0906296fc2e811218f86a4ba9b8dbcdfbc466389c3fa10d3c7bfda2fafb8e2f4","result":{"confidence_high":98.12685329553614,"confidence_low":89.65092448224163,"sample_count":45,"score":93.88888888888889,"score_display":"93.9","source_stated_rank":51},"run_fingerprint":"15f25f21eccd7928f666384628049ce2ab04a3eb1329715dac9127d213b578e2","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":39,"at":"2026-08-27T13:29:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T13:29:52.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0aca063d8106f84918bc0f03eb21f47d28708cf22beed3283393d118caf63c36","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5578947368421052","leaderboard_private_problem_count":285,"model_release_date":"2026-08-20","public_example_count":10,"stderr":2.946997752822398},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"YC2u9wykbYQg4gdkf76j26","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GLM-5.3-Flash (max)","thinking":null,"upstream_model_id":"glm-5.3-flash_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"38abbc3adc3f7b833dbc0883b188a1ffc38ddc878e9e36dff4d67f43e808a1af","result":{"confidence_high":61.56558927974242,"confidence_low":50.01335808867862,"sample_count":285,"score":55.78947368421052,"score_display":"55.8","source_stated_rank":41},"run_fingerprint":"0a70632eef4f9144af5ca2f7778f77a0f2a11513bc366e5c9bc33f6ee61fd406","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c9aa877c028cd97d9e7236cca57e779c43365e21dde1f033dcf2d8c8dc0e5d16","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.09331568458333339,"model_release_date":"2026-08-26T00:00:00.000Z","results_url":"https://arcprize.org/results/zai-glm-5-3-flash"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"zai-glm-5-3-flash","model_type":"CoT","upstream_model_id":"zai-glm-5-3-flash-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"43a790acf37c517ffe418e9a2ce33e021e3dd50d977f332795071689b4f024d7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.83333333333333,"score_display":"66","source_stated_rank":65},"run_fingerprint":"1a3127f2a8f079714a8ae2cdb35914dfa9a7ef3ced4db8e6c24aea6d23570070","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e779e52055420d70dd4daa0d2c386e8be98389f82231c1e7df1fded8217891c0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.07734862875,"model_release_date":"2026-08-26T00:00:00.000Z","results_url":"https://arcprize.org/results/zai-glm-5-3-flash"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"zai-glm-5-3-flash","model_type":"CoT","upstream_model_id":"zai-glm-5-3-flash-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"eb0e49429ef91d13dbeaaa5565d91efb37a7f4ebbcaf982aa7d165924cc0016d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.138888888888886,"score_display":"50","source_stated_rank":96},"run_fingerprint":"f8e6b0842847cc26361bc1330c95337947672a0a059eae10f09900b4ba5b2127","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_549ab507d1f6fe5b6e3cb294","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"83f3473e8de60a9ca93efcd9456daef81ba402818cfc4a5f4df5077946c4f711","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.05822348833333333,"model_release_date":"2026-08-26T00:00:00.000Z","results_url":"https://arcprize.org/results/zai-glm-5-3-flash"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"zai-glm-5-3-flash","model_type":"CoT","upstream_model_id":"zai-glm-5-3-flash-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5dbc8e64a4a3711ad059203a56774817ba9899f96361daf2a1897ec5fb0311e3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.916666666666668,"score_display":"28","source_stated_rank":122},"run_fingerprint":"2b25057873185fe1b5b343f0cda5bf8920170b9aec33e5734f62b08cbb07742d","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d052ebd297c2b24e18dafe38","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":39,"at":"2026-08-27T13:29:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T13:29:52.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4917433b686e3112d2829eb5b821dd26f86cf5503e54fa9de9b8de1674eb0b8c","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.17073170731707318","leaderboard_private_problem_count":41,"model_release_date":"2026-08-20","public_example_count":2,"stderr":5.949419959829496},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"iCysWvFJevNyw2dbYosU6J","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GLM-5.3-Flash (max)","thinking":null,"upstream_model_id":"glm-5.3-flash_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"f8aa604bd1c0f91a30ac21123b2b364ca02105a735bd72c50c5f559b6a2d2ed3","result":{"confidence_high":28.734033852973127,"confidence_low":5.412307610441507,"sample_count":41,"score":17.073170731707318,"score_display":"17.1","source_stated_rank":48},"run_fingerprint":"b9d1893a9966f6582dc994dccc996343408b74fccb590407f5c6003233105d6f","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cb44561203b5d30b7cd30f44463e5f839b95e67c839ca18a74dda6472485ca62","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.000962,"date_is_proxy":true,"latency_seconds":28.218,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.834},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d276717bb8ae4b1f59e40999aed6dc0065a22f4f7569b3a63f61b111409a431e","result":{"confidence_high":87.64664,"confidence_low":84.37736,"sample_count":null,"score":86.012,"score_display":"86.0","source_stated_rank":23},"run_fingerprint":"121028494857e52b7fa8db62bb1f485db6d0fdf256531aa4f7f0f96e2c1f5d10","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"24bde7f80f2ddb5f2478662038591d4a005fa5409ff1abdc2af9fe95cb1ee7a1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-20","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.3-Flash","source_effort":null,"source_model_version":"glm-5.3-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"cda300ed2945c80bbe368789b74d06a20a060a0ecff520d376230e8f02e0de36","result":{"confidence_high":25.577174,"confidence_low":8.829054,"sample_count":71,"score":15.4285714285714,"score_display":"15.4","source_stated_rank":75},"run_fingerprint":"2b6414a17de298361a43d06273eb6094a9a92751c0cf9ac58d8637558cc1f933","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d32f80db8f8edf3a161d1982","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"678ed1c6d73c711b12035c264a75c6d400f43bb87616179429db95d532c643cf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · GLM 5.3 Flash · critpt"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-flash","upstream_model_label":"GLM 5.3 Flash"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"466495ddb9ffae0debeb902af335e98e7ebd85aff079df09071d2d49284d17f0","result":{"confidence_high":25.660281,"confidence_low":8.79401,"sample_count":70,"score":15.428571,"score_display":"15.4","source_stated_rank":null},"run_fingerprint":"e9a028e4b33186345ed6c01256c4e1aba496058867e436bf983dfc704143ee85","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM 5.3 Flash · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_626f9b1c4e05a66a41042aa2","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":39,"at":"2026-08-26T22:26:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-26T22:26:55.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a78e0a3a789eb776e06d53d7b6f1d29ea86232938341acd5d8a392eedb925b5e","metric_details":{"best_score_across_scorers":"0.14","model_release_date":"2026-08-20","stderr":3.4873508801977695},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"b3cBsiDi4D7NtisdffQqxV","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"64b448d4156184a1c02b9e45a5be88e992ad3176a83d65b675e2c3e26b995c99","result":{"confidence_high":20.83520772518763,"confidence_low":7.164792274812374,"sample_count":100,"score":14.000000000000002,"score_display":"14.0","source_stated_rank":113},"run_fingerprint":"53bca0827e8fb50d8b3ee73a95799e9fd0f2dcc8af5eab2fc0f3e8e2405490b7","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8c8b14c68165a9a1385f94b3ec1f730ac6c80fb27df209cee597009f079f41df","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.017996,"date_is_proxy":true,"latency_seconds":3817.379,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.214},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3-flash"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"e490b5448f7bd716b7370d0309fd3e23f67640e0274cfab234f8cc2d01a3ac38","result":{"confidence_high":94.37944,"confidence_low":89.62056,"sample_count":500,"score":92.0,"score_display":"92.0","source_stated_rank":10},"run_fingerprint":"fc83cf5c7ad8276d1b7586460f996a6370b88161a5135930469a1f2520f34a83","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"cb923b3372dc619c5d97dd0cace2f1ab6e2ed79fc013843aef22c7e82e7a0568","metric_details":{"ci_attempted":448,"ci_half_points":4.377970751693644,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":284,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":122.890625,"mean_cache_tokens":11770874.857142856,"mean_cost_usd":0.48196371245535713,"mean_duration_seconds":1537.7995419129463,"mean_input_tokens":12387025.401785715,"mean_output_tokens":72829.77008928571,"median_agent_steps":110.0,"median_cost_usd":0.389975125,"median_duration_seconds":1322.635727,"median_input_tokens":9358476.5,"median_output_tokens":67491.0,"median_output_tokens_to_pass":68067.5,"median_peak_context_tokens":136183.5,"n_attempted":448,"n_passed":284,"n_tasks_attempted":113,"n_tasks_passed_any":96,"pass_at_4_points":84.95575221238938,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_glm_5_3_flash_max","source_model_version":"glm-5-3-flash","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"600d6a11dd0c7650a51ed4c33e6676ad6579879e8b378aea81b58fa00e0ae39c","result":{"confidence_high":67.77082789455078,"confidence_low":59.0148863911635,"sample_count":448,"score":63.39285714285714,"score_display":"63.4","source_stated_rank":30},"run_fingerprint":"23a44ecb25a1383a9d24b3ad23ce046ef78317b1f2e423d712186d7a1f15d39f","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":32,"at":"2026-09-02T21:26:37Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:37Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a82f1a8b12a8d7779677572ec4f07cd7e0695dc75ef7aa570084b716308c218d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"122.890625","mean_cost_usd":"0.48196371245535713","mean_output_tokens":"72829.77008928571","model_release_date":"2026-08-20","notes":null,"pass_4":"0.8495575221238939","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5-3-flash (max)","source_effort":"max","source_model_version":"glm-5.3-flash_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"c5c34de2030ec23e082681722b559a47e5da397cd2c24c7161f21d06412cc144","result":{"confidence_high":67.77082789455078,"confidence_low":59.01488639116349,"sample_count":113,"score":63.39285714285714,"score_display":"63.4","source_stated_rank":30},"run_fingerprint":"b08d846989fbbbdb9395444e21d742a497e33a9dc2ca7da792b89158bfa54351","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5d991ceee2c097f7b84788c4c4aefddbc6deaeb18003799016e3a3b905e313fe","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-20","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM 5.3 Flash","source_effort":"max","source_model_version":"glm-5.3-flash_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"chisel","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"e154a6e39e275dc6b333833d352adbbf17476fdd019dba17415fbac9e84929f6","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":31.830000000000002,"score_display":"31.8","source_stated_rank":26},"run_fingerprint":"cab9421a8ab397c536299a40c14bd7f1012f79babbb967f79ec1c13f058e0a24","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1cc9beb9d89b12c8ba6b20cfc9a160f560c96a976793621658f94b6dd40f9a2c","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.429263,"date_is_proxy":true,"latency_seconds":5548.524,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.619},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"f214da03a298d785b6a70f1a4a13a8e4c95585f78c540d4cbc1d7494c305c876","result":{"confidence_high":57.63324,"confidence_low":47.36676,"sample_count":null,"score":52.5,"score_display":"52.5","source_stated_rank":21},"run_fingerprint":"d430c24db22375a1a31b6574b810bf8f0accae6a26c1661f48169cc8f1695c57","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"eae13894a1e5f6bdeb525602f811e1d810c5270c2a817161a33323f383a8d0ea","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.007053,"date_is_proxy":true,"latency_seconds":301.901,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.051},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"max","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"dd4453f30cedb3a4ce171ebe40b4f142395c208bf5a995b8226d40aa6e8788c0","result":{"confidence_high":82.56996000000001,"confidence_low":78.45004,"sample_count":null,"score":80.51,"score_display":"80.5","source_stated_rank":69},"run_fingerprint":"75150a035c495701d09c211105a369610e684fde4115cf0db34f8bf943833954","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cd5d8da8353c72084873adf2b71d5e6a2a5f4c30573e1a3d7e70f161a4634b64","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-20","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM 5.3 Flash","source_effort":null,"source_model_version":"glm-5.3-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fe3812cff5c418b06450b22315b4f9d9b329cf66f0d88df9602ab042bb02931c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.6203703703704,"score_display":"51.6","source_stated_rank":75},"run_fingerprint":"cb38752d8b9110d9ace9f27e019502daecc3fb9ac4d0b95528a5e47385f58441","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7f5a892f8608b1bcfaf9c196","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0247114aa31f154d2a93485c8031de2077f7ef3b2872394c573d682792e250bc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · GLM 5.3 Flash · scicode"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-flash","upstream_model_label":"GLM 5.3 Flash"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"215c31c0965041ef18468dd0216e2d8af1556c9ecbf366c9dbe20e55516aac01","result":{"confidence_high":57.332639,"confidence_low":45.865442,"sample_count":288,"score":51.62037,"score_display":"51.6","source_stated_rank":null},"run_fingerprint":"35d41d75f7bae255f4c8fc0529dcaffca1f12442bf10a79625bf24592ebc4e0e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM 5.3 Flash · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_626f9b1c4e05a66a41042aa2","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:35Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:35Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"14c05d7a603cf09794f4e57c228e2b48d12ef0b8564bb167945e326a2159af82","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.010869","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-20","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5.3-flash-high","source_effort":null,"source_model_version":"glm-5.3-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"871e7a4ce82d510c855fb65d3bb95b88e7fc08a92581aab4e2f537171b51df37","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":303.55,"score_display":"303.55","source_stated_rank":111},"run_fingerprint":"1365451ae699c0517736c5a86b64a0890e1a26ae3d1381f1166e0ec4290956a8","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_549ab507d1f6fe5b6e3cb294","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"506c4d2891c656e8dd193d88b1c7e68a8226b5dd151b5d4da6a9190f1fbfdb0b","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.348805,"date_is_proxy":true,"latency_seconds":5027.281,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.277},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":131072,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3-flash"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"80494d835c8e371dc2732200709a7bc962c886dd07469523c2e588d7c70c5b5a","result":{"confidence_high":41.10192,"confidence_low":20.41608,"sample_count":null,"score":30.759,"score_display":"30.8","source_stated_rank":67},"run_fingerprint":"486b9b243ca8bcb6d24b8813fc30c1fd8378e3653f4abee39ff0825450ca9daa","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a7b778e75f8d2e6b805181ee4bbad3aa0528af9c11000fa3f2999ced55c1d1d0","metric_details":{"license":"MIT","variance":14.816828425062418},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"glm-5.3-flash","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8e1f3877a4c3997403f7d95c130daef3f638171f6c3bf26d21913271871d56f0","result":{"confidence_high":1623.110237867913,"confidence_low":1608.021402757386,"sample_count":10355,"score":1615.5658203126495,"score_display":"1616","source_stated_rank":25},"run_fingerprint":"04b5cbe0ac3dd3c044f2ac883db8f32749206a8b77377049d5942ddaa1c1e34e","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d35421dc8bde7a8d07e82ffb","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3427358fae5bb33ef30e631234889613ba116e020069faafae144f08ae6e94cc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-20","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.3-flash_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"f018191e3e0ccbd94a6e46db7694766b170af58402ca21479fac5df89a56d666","result":{"confidence_high":1619.06,"confidence_low":1595.13,"sample_count":2925,"score":1607.09,"score_display":"1607.09","source_stated_rank":20},"run_fingerprint":"8de39e288c5850c13a84b45a37b0361b8ff81c50feb33fcba7a83351d2b51228","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_918e4f821ea6d4585f7a5c86","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6d61b2e63e01799d2a81dd928cc3830a62ac1aaee777ba65e006eb36b4f260e8","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.028403,"date_is_proxy":true,"latency_seconds":965.737,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.247},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":131072,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.3-flash"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"cf2fc7e80a1487c6b0cf2c323b4cdd2f98fea33c8815dd83da678feabe331181","result":{"confidence_high":67.32512,"confidence_low":58.51688,"sample_count":null,"score":62.921,"score_display":"62.9","source_stated_rank":38},"run_fingerprint":"13e742660319b2642ea8dcc664e2e6488d22b2d84a34f21c691a347a66363376","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_d3256297c99cce45effb308c","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"93f906842a4281ed7919370bc404687b2b24bfab43f1780bc8014657a979433f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.3-Flash · terminalbenchV21"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-3-flash","upstream_model_label":"GLM-5.3-Flash"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"43ca4a384c39e2a683b07d3e694d152abc464dd49726738c4f07617a487973e4","result":{"confidence_high":90.392278,"confidence_low":75.311028,"sample_count":89,"score":84.269663,"score_display":"84.3","source_stated_rank":null},"run_fingerprint":"095024bef292fcf549eded45c496ce4206584f747e2ad0cae9528ff06ed9f5a5","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.3-Flash · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_626f9b1c4e05a66a41042aa2","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"74fe45cf1a49a37745f8df95e0f29f80e24bd3b9342b66b299fd988bd3ebd7b7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · GLM 5.3 Flash · tauBanking"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"glm-5-3-flash","upstream_model_label":"GLM 5.3 Flash"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"494ec0689d93159ebb4221c9ca0b2310957c285dad9cfc4e84578ced0b38bf51","result":{"confidence_high":57.066997,"confidence_low":37.578071,"sample_count":97,"score":47.216495,"score_display":"47.2","source_stated_rank":null},"run_fingerprint":"865973baaf1212feefa0f054fe10e3e15bf5f55882e4a6a7533cff0a0501dba8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM 5.3 Flash · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_626f9b1c4e05a66a41042aa2","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"edc7268bb32e1fc996318831e92d92526e04776a9965a7815a66b0f2c1a2a52e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.668","mean_standard_error":"4.4","model_release_date":"2026-08-20","notes":null,"standard_error_points":530.0,"upstream_source_url":null},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.3-Flash","source_effort":null,"source_model_version":"glm-5.3-flash_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"88b3e656b24dfd4b5b94cdb91806133587724f6eef37624fba4b12cdc15a9c41","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":52.800000000000004,"score_display":"52.8","source_stated_rank":19},"run_fingerprint":"469e65f1c2f31a46640ac53bdb2e24bede6fc3dd57459826fd9a9c5948f6c513","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_605f0eded166b30ff74e07bd","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aae7a6f54135d7f2f709e408dd4dade5b960c51f123a59dbebc406a771d33b68","metric_details":{"confidence_level":0.95,"license":"MIT","session_count":80836},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GLM 5.3 Flash","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3112cf499393bd590953c2f38d8f20baaef2ddfa4c0b08eb3ef59807d14ab704","result":{"confidence_high":0.00030784483810813047,"confidence_low":-0.009040788597539741,"sample_count":9533773,"score":-0.004366471879715806,"score_display":"-0.0044","source_stated_rank":32},"run_fingerprint":"a3e066d9dd5cf729cf36a1b67187f79fe6ac7af9eba3f4c8a1163c7065835737","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9b74d27daa9d9ff35df369b7","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fcbdadb8b5690a50c3551e46c90d3819a84cedfe4790cd59f7e4d66cc0b9bb9b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","reported_confidence_interval":"-19 / +19","source_locator":"Leaderboard models table · GLM 5.3 Flash · gdpval"},"model":{"id":"zai/glm-5-3-flash","name":"GLM 5.3 Flash","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"glm-5-3-flash","upstream_model_label":"GLM 5.3 Flash"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"99d682339d63832cddb842b37b286e016b2e496f5802cf11be7e076188074cdf","result":{"confidence_high":1665.4,"confidence_low":1628.16,"sample_count":220,"score":1646.78,"score_display":"1647","source_stated_rank":null},"run_fingerprint":"7e525b300dd8b8bed62889057f83bba5156df704bc4bf3ae5d5da8bbd4c3df11","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM 5.3 Flash · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_626f9b1c4e05a66a41042aa2","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"94d82032ecdf7d3aad5ac2cab3d7ccef044ea783bf5bef65e5c5f692684b3e0f","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-12-17"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 3 Flash; model release: 2025-12-17","source_accessibility":"API access","source_model_name":"Gemini 3 Flash","source_model_release_date":"2025-12-17","source_model_versions":["gemini-3-flash-preview","gemini-3-flash-preview_high","gemini-3-flash-preview_low","gemini-3-flash-preview_minimal"],"source_reasoning_efforts":["minimal","low","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"61662f3c34b9f8c353a6a470aa9fd4e5eb50624f640641b6c03bac77a2c087a8","result":{"confidence_high":153.73,"confidence_low":150.39,"sample_count":null,"score":151.82,"score_display":"151.82","source_stated_rank":44},"run_fingerprint":"777a807ca767736143cc955c7c9e4ddad8bae534132ce413f120a31b21906c16","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9ffb97cc9acccbbc83ffb92e","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a26db8b49af4762f6a8c2150fb1a7ca5ae9e25094db5fae402417b6399964120","metric_details":{"license":"Proprietary","variance":2.3876753013458214},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1461.2419532874922,"confidence_low":1455.1848396688422,"sample_count":90842,"score":1458.2133964781672,"score_display":"1458","source_stated_rank":61},"run_fingerprint":"b3fef0cce951572bb25de61dc40354203e119efbc163cad28a2a65e1bb86a9bb","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_57d19654e2ae916ecd0b5c09","provider_config":{"source_label":"minimal"},"provider_mode_key":"Minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"785f8fc166959dd7f928a32cfa02e6408ac71f8da949b433066b9a528a50a0a9","metric_details":{"license":"Proprietary","variance":4.920414770224555},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1477.0762681802148,"confidence_low":1468.381080839955,"sample_count":31259,"score":1472.728674510085,"score_display":"1473","source_stated_rank":42},"run_fingerprint":"c43cdd1d8d83dde148f8d1fe06d12dbfea24c79efc2dc14bb8fc1ea1140bf7cb","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7355d5bade9de2a9b697d23c","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":292,"at":"2025-12-17","basis":"evaluation_at","evaluated_at":"2025-12-17","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1328f88b0775ea781b01ca80ddfee244a6e2e349952204872c2029a0deb25c06","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/google-gemini-3-flash","source_model":"Gemini 3 Flash"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"26c8fd00871cbe3c4d547eae32d78e6ffec3b09798c45289975fdd0b75864645","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":36.6,"score_display":"36.6","source_stated_rank":18},"run_fingerprint":"3363893cf9e742cee93dda00777449346362ce7e2e56df2d111008ceacb3734e","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a37ea0078b8bcb4cc5bfbaf0","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:30:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:30:06.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3b1a70d9ba4377348e7f6eb28587d34de1226ce522ef16148e96e370a653fae7","metric_details":{"best_score_across_scorers":"0.668","model_release_date":"2025-12-17","stderr":1.4899597242811566},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7cc6cDCXjsPjtKt36bbeMR","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7cc6cDCXjsPjtKt36bbeMR.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/7cc6cDCXjsPjtKt36bbeMR.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"05f29e9b0116a60ecf7ad7e2990a0fdde7e3db4fd096daa36075437027847a71","result":{"confidence_high":69.72032105959106,"confidence_low":63.87967894040893,"sample_count":1000,"score":66.8,"score_display":"66.8","source_stated_rank":10},"run_fingerprint":"556939c99adee9508cadeda02dc602d9e9c3366dff0e6be1bab01593321aa99c","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7911225912f5adb57d2ba51803c3359996fce406bcaa05e1c220b32c89e7adfd","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.044346,"date_is_proxy":true,"latency_seconds":83.84,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.638},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":86.364,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-flash-preview","zero_shot_accuracy":89.394},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"d712c53ed29213a5d246b3989f00a2e8c25486ebe5aa9be6aa62ad43cef7846c","result":{"confidence_high":91.08948000000001,"confidence_low":84.66852,"sample_count":396,"score":87.879,"score_display":"87.88","source_stated_rank":37},"run_fingerprint":"ea631166eda9043411c0d1f4e8b7d65c51df1f6167b90536aa993916dd09aa5d","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:22:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:22:58.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ec8d81c6b129c25fc2a7fc4e555abdf86714adf329454933b44c8e6b7f1bf5a7","metric_details":{"best_score_across_scorers":"0.8939393939393939","model_release_date":"2025-12-17","stderr":2.1938047738853106},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gVdraDT3pXmaExP9VbqME4","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgVdraDT3pXmaExP9VbqME4.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/gVdraDT3pXmaExP9VbqME4.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6945389d9b90cb09c2ec988c47d3d4bfe881ebf67c3084fd1a02ddde1fcc1451","result":{"confidence_high":93.6937967507546,"confidence_low":85.09408203712418,"sample_count":null,"score":89.39393939393939,"score_display":"89.4","source_stated_rank":51},"run_fingerprint":"341e27ac784f363b44119696982df6faeb799344ea4ecc40102488bdf67f94a0","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":291,"at":"2025-12-17T22:39:00.278Z","basis":"evaluation_started_at","evaluated_at":"2025-12-17T22:39:00.278Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b8ce9d25b06d0e59fe5eef9ed51069aa0639696436a019cda6451a504ec4e059","metric_details":{"best_score_across_scorers":"0.8320707070707071","model_release_date":"2025-12-17","stderr":1.9036353018587762},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"oAu9CZSNPZgwiwKnoNpM8Q","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoAu9CZSNPZgwiwKnoNpM8Q.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/oAu9CZSNPZgwiwKnoNpM8Q.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"4caadceab8097d648bf57085c6a6f895ccd8f5e4479fbf1a475cf45f4b3bede9","result":{"confidence_high":86.93819589871391,"confidence_low":79.47594551542751,"sample_count":null,"score":83.20707070707071,"score_display":"83.2","source_stated_rank":110},"run_fingerprint":"0b7de76010346824699f982fbb652e37674a89948b7ddbfdfec44dd423d8017e","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8f8424a4c94b8ae01b77e76d","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":98,"at":"2026-06-28T14:53:14.029632Z","basis":"evaluation_at","evaluated_at":"2026-06-28T14:53:14.029632Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"d02d37f0179ffe59f4380b0f98c1a7bf14734e38260293c22f720f517dae8e1a","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3-flash-preview","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"93e10680576df054bfa8c2e7b50b9de0a1ef3e58855e4874c0781ec431f4d927","result":{"confidence_high":94.09634105718209,"confidence_low":85.7016387407977,"sample_count":null,"score":89.8989898989899,"score_display":"89.9","source_stated_rank":7},"run_fingerprint":"70faa8ecd2f9e4b1b1406025c40320876d80eb56a172866af0fb0bc89b2c08d1","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_665acd0e6a4e4f0162bd3ceb","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"704037ee7941ea5e33e161ed4e5339b3cf4febcb130eeafb482f959cda100820","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.019364,"date_is_proxy":true,"latency_seconds":36.249,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.315},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65536,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-flash-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1dfb3e838f5c68a73605127744d3a612e724036fe31abef342882505ce62d105","result":{"confidence_high":89.2094,"confidence_low":87.9746,"sample_count":null,"score":88.592,"score_display":"88.6","source_stated_rank":19},"run_fingerprint":"2e117a8ecb5ee45fbdb313b7da5a4e128b9071a9e33a158cbb345267d18a02ca","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:25:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:25:57.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9560c41d165042515772d1bbe0ddf7d367c77792748e187a7fe9ff9356822723","metric_details":{"best_score_across_scorers":"0.9555555555555556","model_release_date":"2025-12-17","stderr":3.1067790907534745},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"a4uE6KgMhGSUjcnemRnbkA","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fa4uE6KgMhGSUjcnemRnbkA.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/a4uE6KgMhGSUjcnemRnbkA.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ee7434d0c3b0d41a21b877f0967b93eddf632c1e85f5c451cedbf79f383c609d","result":{"confidence_high":100.0,"confidence_low":89.46626853767874,"sample_count":45,"score":95.55555555555556,"score_display":"95.6","source_stated_rank":38},"run_fingerprint":"88020f6d00fa4ed429b60883a8705dae9ebebaf24607801751199242c16eba16","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":291,"at":"2025-12-17T22:40:30.896Z","basis":"evaluation_started_at","evaluated_at":"2025-12-17T22:40:30.896Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b0e2c28fbfc94f2054ff494e1bd80cb762f8c1b742070c1f8e96544e515bf69b","metric_details":{"best_score_across_scorers":"0.9277777777777778","model_release_date":"2025-12-17","stderr":3.5196259111492543},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"H3E7EjjXfGCPnm9ooLdmAo","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FH3E7EjjXfGCPnm9ooLdmAo.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/H3E7EjjXfGCPnm9ooLdmAo.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"86f1a8a5868416ce9d5d00c05afa619d56b9a480032bb489f142e3e20d3fe6da","result":{"confidence_high":99.67624456363032,"confidence_low":85.87931099192525,"sample_count":45,"score":92.77777777777779,"score_display":"92.8","source_stated_rank":60},"run_fingerprint":"9de013d49a7729f1816af57962c4b2b09e19b0ff38fd05b1236b31a2286ae0da","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b8a972f6855beafd08a430ca","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T23:44:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T23:44:30.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1aa2cfa4de9fb4821fbff8ace2ec9f8236c97aac9ca5e8f194ed9c9fab5d4fe4","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.512280701754386","leaderboard_private_problem_count":285,"model_release_date":"2025-12-17","public_example_count":10,"stderr":2.966059084324674},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"EtgrDq6ei6kZe8K7QbXUe3","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEtgrDq6ei6kZe8K7QbXUe3.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/EtgrDq6ei6kZe8K7QbXUe3.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3 Flash Preview","thinking":null,"upstream_model_id":"gemini-3-flash-preview","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"96cd26da29a9ae656aab7d662b37cde34e048e9768f2ea03150eb5a79fcd803a","result":{"confidence_high":57.041545980714965,"confidence_low":45.41459437016224,"sample_count":285,"score":51.2280701754386,"score_display":"51.2","source_stated_rank":47},"run_fingerprint":"b2c535fdb0a2ae338e6b615d9dacb7be6f298b22defb348e9efd688cb915a662","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0ba24c793659a66d8cbdc90d","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"833020745bc1430de13fed8ebd5f72e22b4ffc01b9909e11491888a6cdd32824","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.2314,"model_release_date":"2025-12-17T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-flash-preview-thinking","model_type":"CoT","upstream_model_id":"gemini-3-flash-preview-thinking-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f8312f452c1590fb2d8d5d0058ebce8a3f1cfed597e44f9ed6968b16bd4b9dfa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.61,"score_display":"34","source_stated_rank":110},"run_fingerprint":"d9fbb13b73be6d66202aff19b736e935b38a490b5648c0a8e6a05dcfeb00947e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"10ac5ddc696c2073d23f0637c813da812dd8db933744987225390f0b42014b5d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0824,"model_release_date":"2025-12-17T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-flash-preview-thinking","model_type":"CoT","upstream_model_id":"gemini-3-flash-preview-thinking-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e29e385401f6bda6b39837fe6124ab22b37728ff04aeacc9be2c17281473e22d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":12.78,"score_display":"13","source_stated_rank":140},"run_fingerprint":"239431e43ae55bce35490b5484a9fb8379f44ce8e0508c9923634fe553fb1659","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_26b891d93df44a20d111d745","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"90c6f9f13216d5e1f72f90597e511a011a907246d63c2bcc3b1bfc05c0f5229d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0249,"model_release_date":"2025-12-17T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-flash-preview-thinking","model_type":"CoT","upstream_model_id":"gemini-3-flash-preview-thinking-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"69d32cf4f50940ef6be93009ead1e237a31b973496e5db550704b0cdb736ccec","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.25,"score_display":"1","source_stated_rank":217},"run_fingerprint":"74afa1008ac480b0bf6b8a5174bf9f700a027392b3a2e52c37b8da05342818f8","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_43de39138d3dd71be31a63c5","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"89d0e5be0cf9007adf07fb3fcb3c4bbdffc0def500a3b6d60e96835062e0efb3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0205,"model_release_date":"2025-12-17T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-flash-preview-thinking","model_type":"CoT","upstream_model_id":"gemini-3-flash-preview-thinking-minimal"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1932d5659850ee0f772236bbf1f8d48d31bf06563d8de2a2c6100cc11f6d36fb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3.3300000000000005,"score_display":"3","source_stated_rank":181},"run_fingerprint":"d61f7ccf8ce525fbe040e9d26d99a31729589ff101c063ecb09bf0b3a3ea18c7","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_57d19654e2ae916ecd0b5c09","provider_config":{"source_label":"minimal"},"provider_mode_key":"Minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ca8e2233c0e5475f2f050dd7229d50f0b6eabf24f6e522bf0efde927067a5e35","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash Preview (High)","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"150e6a848937a732a9fdb3f828344bd367b1cf14b1658d90aff5ee1afbab6995","result":{"confidence_high":38.639977,"confidence_low":28.926129,"sample_count":360,"score":33.61,"score_display":"33.6","source_stated_rank":111},"run_fingerprint":"8c25263715306c1e578b6946db25b09f09cba34031d76d999697ecc597a448e2","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b823ab05ddf702241222885fba0f49000cb07c94feb964ea0517e17446d1053c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash Preview (Medium)","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"19b4b1a1d9559ed679b0173d88828c519f5ce41c99a38947d6b547b4bd8ec9ef","result":{"confidence_high":16.626046,"confidence_low":9.719924,"sample_count":360,"score":12.78,"score_display":"12.8","source_stated_rank":136},"run_fingerprint":"9aaad4449c6e66c94f4256b0b338919325de73f22c09ce5dd0c0600174959f9f","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_26b891d93df44a20d111d745","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5dc0adb466770a260a99440df098fa612689f4309045b1d40af98f53dd884d25","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash Preview (Low)","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3c903c0a983d892c14e9f4e4f84e621c91f16ddf68a205c1c4123cc1632ccc13","result":{"confidence_high":3.017021,"confidence_low":0.512427,"sample_count":360,"score":1.25,"score_display":"1.2","source_stated_rank":209},"run_fingerprint":"c6696e3939ff73acf3b91f563c450818b70ef80891c598a8a5511a22d8181621","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_43de39138d3dd71be31a63c5","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"792c51f7e2d6a487aba546f298ae682af6ad5de160fceaca2e8f60139c381c1c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash Preview (Minimal)","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"44a28edac2c4447fad5ac651787e97dffe663e998ff74f45685b8a0893f98715","result":{"confidence_high":5.731085,"confidence_low":1.91444,"sample_count":360,"score":3.3300000000000005,"score_display":"3.3","source_stated_rank":176},"run_fingerprint":"d293f436aa1e00ecf621c1dd2c7e17c7d6c15acc7eec215135b143e8d9204edc","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_57d19654e2ae916ecd0b5c09","provider_config":{"source_label":"minimal"},"provider_mode_key":"Minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_3442b9e1c78d3919dff3a62a","split_or_window":"semi-private-validation","unit":"percent","version":"ARC Prize verified semi-private"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":139,"at":"2026-05-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"41495e3d875a5136251ae4b6aa82b3afe3d9abe117d9a85267156037149f8c0c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · ARC-AGI-2"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"ARC Prize Verified; Google Model Card re-report","reasoning_effort":"reported","result_published_at":"2026-05-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"ff0a67497c6d2377c884521186b2e9dddc4cf6e001e044d02319150159a2f674","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.6,"score_display":"33.6","source_stated_rank":null},"run_fingerprint":"5ecc9788a31de7d048e44cc71ddab69884c652acfcd669d157fd46e1fdc214dc","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · ARC-AGI-2","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fee9b688e55e7a2cfd3af24c","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T23:44:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T23:44:30.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"da0cf658aeb7a4a89ea77d57de6474c937842cc2b7ff83d31583f5435b67f5d1","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.17073170731707318","leaderboard_private_problem_count":41,"model_release_date":"2025-12-17","public_example_count":2,"stderr":5.949419959829497},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"ZczEhu69FQ9Cnns8NT6ew4","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZczEhu69FQ9Cnns8NT6ew4.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZczEhu69FQ9Cnns8NT6ew4.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3 Flash Preview","thinking":null,"upstream_model_id":"gemini-3-flash-preview","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"0ff253f7b5fd2063e6cb521ab31fc665724c492b9c6f1484984fbe7bddeef367","result":{"confidence_high":28.73403385297313,"confidence_low":5.4123076104415055,"sample_count":41,"score":17.073170731707318,"score_display":"17.1","source_stated_rank":48},"run_fingerprint":"00726ab31415552987a56af5ab811e8db66475462907062ba85e6cfc407d0a54","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_92976465691e2c940c861d92","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8a54d25b3f226267b7ceefb2c3e6a9179e43bb610ce7c7340f2303c1d24347df","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.014422,"date_is_proxy":true,"latency_seconds":27.856,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.792},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65536,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-flash-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"cbac24a04edcc4666fc0c6964d65ef0c0a5b9b9110099e64a5a799d5481bfe80","result":{"confidence_high":89.18231999999999,"confidence_low":86.07768,"sample_count":null,"score":87.63,"score_display":"87.6","source_stated_rank":13},"run_fingerprint":"f9a1c17694f64531e088c6292c9050c87e0a32e57c22bb763da90e2c0dfd1d09","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_39a5bff9a48a29bea5ce760d","split_or_window":"standard-no-tools","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":139,"at":"2026-05-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"49379e3fdd983a9736f298f8b523d4a545705ce2d84b789d4599e9be110ce34d","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · MMMU-Pro"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider evaluation; standard and vision average; pass@1","reasoning_effort":"reported","result_published_at":"2026-05-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0fe020a8765804804ad5ddbf01716ecad56c8a711c41eae06fffe67f7933b97c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.2,"score_display":"81.2","source_stated_rank":null},"run_fingerprint":"12b7f1179821634e259ccd9b393b513e2aa34092c923491ad8b60b969ab19878","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · MMMU-Pro","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fee9b688e55e7a2cfd3af24c","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8b96277c5eb5c75349ed30194483d49b05c6e2a1ceb70cd86277f92584e31a45","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"3314247dc49fdff2278501e4ff87391a4638f65db8f3227c6256e5e39a236aae","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.1,"score_display":"61.1","source_stated_rank":27},"run_fingerprint":"376f1ddff5387cb480dcfdf1caf32a3996273a0b9d3d2a5f9a5769e3e827f205","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d6963c4fa76fa587ee8fc818","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:22:33.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:22:33.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bc9c35f191db9f5941e473a45ba8235b8b915c3d0155f606bbe2dfb7dc753e7f","metric_details":{"best_score_across_scorers":"0.4","model_release_date":"2025-12-17","stderr":4.92365963917331},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"HVtbACyemDaTPW5XUDC6Jf","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHVtbACyemDaTPW5XUDC6Jf.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/HVtbACyemDaTPW5XUDC6Jf.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b0d7777960fd322807f3fe1ede20b552ff737df334193db739abcb8b69dfc9b1","result":{"confidence_high":49.650372892779686,"confidence_low":30.349627107220314,"sample_count":100,"score":40.0,"score_display":"40.0","source_stated_rank":24},"run_fingerprint":"668bfe61e94076336d32f4f929dc859d3cb79ea01f4cc207d8fd78fc24822dee","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":291,"at":"2025-12-17T22:40:30.291Z","basis":"evaluation_started_at","evaluated_at":"2025-12-17T22:40:30.291Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f0e1f9abbd056b5514569fd033d8ce610b4b48ac01b7e86133626300758acdae","metric_details":{"best_score_across_scorers":"0.38","model_release_date":"2025-12-17","stderr":4.878317312145634},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ZMRExFd2WEEwTRSQPLXXwu","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZMRExFd2WEEwTRSQPLXXwu.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZMRExFd2WEEwTRSQPLXXwu.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"00b15ea691b6aac354f00a4e49efdaf0bb91d54a0f256604018573f87ba9e745","result":{"confidence_high":47.56150193180544,"confidence_low":28.43849806819456,"sample_count":100,"score":38.0,"score_display":"38.0","source_stated_rank":32},"run_fingerprint":"d165504b279f403dee0892e6e6cb288f4e43cfa9debbe6d19b71c87060ede795","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_61586d0191b12ed60d09aa76","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ccb82eddcee7d99cb097d20adc8cd6829ba26185f544c12b146490f19007c456","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.1743","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash Preview (High)","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c5ce19de294519af53926129ed4992f7ed8e37de295da6aa5c8ca2f057ba9cb9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.67,"score_display":"84.7","source_stated_rank":105},"run_fingerprint":"754ef43acaf5ef93c3d8b8fd8a8246eefbf01ba6ebe325034e6b9e2a6e3e322a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"51f9196f7b4908fd62e602477ae7f3fc3469aaaf32696107714c410226a29871","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.0612","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash Preview (Medium)","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a152ff440728e47622b19622b2e5e832c4ce2a0320ee1883621be8d03185c8b6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.67,"score_display":"57.7","source_stated_rank":155},"run_fingerprint":"7a6aadfa1d5deacacd4b7c1a5ca6ead2fb4d3c2b8fda18ae6961301e755667af","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_26b891d93df44a20d111d745","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8d3885db6c68bf4802e234dbbb27191a39878e4436293027fa1bfbca98b302da","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.0163","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash Preview (Low)","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"9213f8794826562e9fdab2dd30471afb4635682557d9e17c711cff4af6708a32","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":28.999999999999996,"score_display":"29.0","source_stated_rank":208},"run_fingerprint":"4e584ec27e37692e34d0323ccfc02130bd56f3febdf23efe8b687566db64c12c","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_43de39138d3dd71be31a63c5","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"80e2140ad473a4406d335b43c54aa9aeb0d46e6e44b90b56773a27c9e211916c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.012","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":""},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash Preview (Minimal)","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3b958649ffdbe4a2d19cfb1d7a61e89c7414970d40d4cdcd553023232c713a93","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":21.5,"score_display":"21.5","source_stated_rank":223},"run_fingerprint":"cda19103102e88bafcda60f78d46f6a85838544a1f74dbaf0e8c1b0a9ae65470","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_57d19654e2ae916ecd0b5c09","provider_config":{"source_label":"minimal"},"provider_mode_key":"Minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0ed3e71b15b4280812231eced94edd7f7a114e0a61612e1099776080c37b7e3b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.280231,"date_is_proxy":true,"latency_seconds":285.223,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.938},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-flash-preview"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"92012925ecc244a90ba4c9864adc64795fe0707967ce154a935f9196ebc1434f","result":{"confidence_high":78.79848,"confidence_low":71.20152,"sample_count":500,"score":75.0,"score_display":"75.0","source_stated_rank":44},"run_fingerprint":"29b8532fc09e6ac1c4ee2e30598b28f15b0c109f60e83dd5edd29cba03912d9d","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":228,"at":"2026-02-18T20:12:36.725Z","basis":"evaluation_started_at","evaluated_at":"2026-02-18T20:12:36.725Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"868f41051796904fbe4fe91374db66125d106de1ff7e2a0769e3680d73de4675","metric_details":{"best_score_across_scorers":"0.7541322314049587","model_release_date":"2025-12-17","stderr":1.9593002139508684},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jxoyMNmTnYirjLJN6wRNmG","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjxoyMNmTnYirjLJN6wRNmG.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/jxoyMNmTnYirjLJN6wRNmG.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"9a23979790aaea538bac219af958f6175a170e8a3263080b55b03bd838d2a455","result":{"confidence_high":79.25345155983958,"confidence_low":71.57299472115217,"sample_count":484,"score":75.41322314049587,"score_display":"75.4","source_stated_rank":14},"run_fingerprint":"f9a1ea0487e470f3e91601c1ee31cfc361a4841b7f3ebb69e939ac55199619ca","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bc30db16551ab7c282485381","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":230,"at":"2026-02-17","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17","status":"stale"},"measurement_id":"498f4e8d51e887cd6c240e548ce48f5545e9e0493fb45ae8eefa9ed08d91a882","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_gemini-3-flash-high","logs":null,"model_display":"Gemini 3 Flash","model_org":"Google DeepMind","reasoning_effort":"high","resolved":75.8,"site":"https://ai.google.dev/gemini-api/docs/models","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: gemini-3-flash-preview","Org: Google DeepMind","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_gemini-3-flash-high/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"77d975eb1c8de3880e972f445fd25ecb0728a35fad6abe72da8ba01ffe153338","result":{"confidence_high":79.34828,"confidence_low":71.858289,"sample_count":500,"score":75.8,"score_display":"75.8","source_stated_rank":7},"run_fingerprint":"a87d1bd16407b1623c517bd67158465eed636897609da1b2b4d124e12da267ba","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b31e2006211423b08d16b59d","split_or_window":"2026-01-01/2026-03-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":218,"at":"2026-03-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ffe691496fde7034c7b5b18ba3e6a219fcbc81444b4106d2378c9f613dcc8ecb","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-12-17","pass_at_5":61.904761904761905,"potential_contamination":false,"sem":0.8471146409170146},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"Gemini 3 Flash Preview__tools","window_from":"2026-01-01","window_status":"historical-post-release","window_to":"2026-03-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"41b89107cff12d693d8637505f026e8beeab77a4802fcd952a837ac613bbdda4","result":{"confidence_high":51.471700007552656,"confidence_low":48.15101061515796,"sample_count":null,"score":49.81135531135531,"score_display":"49.81","source_stated_rank":null},"run_fingerprint":"f995ba60151bf9bbe00b9e42b17e2ca9dcf14edad9220040f25a4868f5e9f279","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e83e7303a3376faa702de988","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":265,"at":"2026-01-12T22:42:07Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-12T22:42:07Z","status":"stale"},"measurement_id":"39df9ca6657b7b5de3232bdc3e116b31b91e580c3bbf4acc2346992a45d84c95","metric_details":{"confidence_interval_half_width":3.55,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gemini-3-flash; createdAt=2026-01-12T22:42:07Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2026-01-12T22:42:07Z"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-undisclosed-harness","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":false,"protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"gemini-3-flash","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576"},"run_count":null,"scaffold":"submission-specific","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"7c9991b0af7cc83cdfd32a9f2ba357551e0285e73e28b4d56cb63462a1443186","result":{"confidence_high":38.18,"confidence_low":31.080000000000002,"sample_count":731,"score":34.63,"score_display":"34.6","source_stated_rank":14},"run_fingerprint":"184b09fc244c1bcab1b19ef834e637107206031a196c2384cfd6aab5d081cc28","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=gemini-3-flash; createdAt=2026-01-12T22:42:07Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f908fa0b3f4fdebe0b9f8fb0","provider_config":{"source_id":"scale-swe-bench-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-undisclosed-harness","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_50545b611e994b43926cee37","split_or_window":"google-single-attempt-comparison","unit":"percent","version":"SWE-bench Pro · Google comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":139,"at":"2026-05-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"e4964c8b1b31917b7ae943d8ae42e8a96c0248afdb5055d564451e2b474f5fb0","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · SWE-Bench Pro (Public)"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google DeepMind provider comparison; single attempt; exact scaffold, task revision and thinking level not disclosed","reasoning_effort":"reported","result_published_at":"2026-05-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Google comparison harness not disclosed","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"f68c03b01f7f9834b7acc612c405b63b4e486116c2485c289edc794e6ed13fb9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.6,"score_display":"49.6","source_stated_rank":null},"run_fingerprint":"aa468e9a2ed3496ed19880e125d65cb3c8898db0bde85e54033796b13a3b0df3","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · SWE-Bench Pro (Public)","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fee9b688e55e7a2cfd3af24c","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":290,"at":"2025-12-19","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-19","status":"stale"},"measurement_id":"6e5472e5a52711f1f1ee61aabe1851d1b76146ad847ef38a1ea35ddc50cefdde","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":"2025-12-19","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"c20751fb7a44d040c598104ce00cc785040a219233ed3f47b0b005fc63dff5d0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":9.8,"score_display":"9.8","source_stated_rank":20},"run_fingerprint":"c155a6c934ffca2c3d8a5e972e1d447911302a7dbef6d910fa9cda39b24b5b2f","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_525bef63a282c86f51e299ca","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f9c39f76106063939be4ebe06e1e926bfcef86e5e8b3fc0b1aacbbd8d5e218d4","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.050525,"date_is_proxy":true,"latency_seconds":91.931,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.004},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-flash-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"3b05c5bc0d1074d493384ec6be12f1d72c89f913ae9130dd79d0f339e21e5a45","result":{"confidence_high":87.55883999999999,"confidence_low":83.62316,"sample_count":null,"score":85.591,"score_display":"85.6","source_stated_rank":30},"run_fingerprint":"c632b83be6d6e4ec0cfe2d37cf8c54eb9fb4e386169308b646669f913f4da855","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e9325fe9aceb361d912cf12b5cc4a658913a0fdfe60a70d51a689ebc412006ba","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.091067","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3-flash-preview-high","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"3b90e109f668919c06399d02a95b3e09bad004e7366346c6cc39d5c44dc328a8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1367.2,"score_display":"1367.20","source_stated_rank":23},"run_fingerprint":"582d115a227cbea79bac0ccb61ca207eb8bc431f47b73e626eec3084b0df4d52","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"94f30470f6531e719658077c85eb72ed1e9a05cd47c7aa70e9bbc4a478ba0e23","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.942067,"date_is_proxy":true,"latency_seconds":806.66,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.946},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-flash-preview"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"82ba0776c4d8e08732bf3b373181221e4ef40f66186441d3fb44bf976be20447","result":{"confidence_high":27.93816,"confidence_low":12.469840000000001,"sample_count":null,"score":20.204,"score_display":"20.2","source_stated_rank":75},"run_fingerprint":"eee952341932833e7004457f4bb7c5ca6d282eea30018d28bcf12f87750e4d19","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bd160a392a15e825cab5046347d843e2e2eac3b525674e79ea6edeeb77fe93dc","metric_details":{"license":"Proprietary","variance":6.987342614751639},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3-flash (thinking-minimal)","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"505c4f2a8fa7275364a8f9e81674ecceebf899096d6f41aa8ba6b1c18f28dc25","result":{"confidence_high":1389.1659599483683,"confidence_low":1378.8041861773095,"sample_count":22213,"score":1383.9850730628389,"score_display":"1384","source_stated_rank":95},"run_fingerprint":"cd187b06f02769eeceffa2d90c304c6ca342c6ef09d4ab6be4b4b6a7eb7a957c","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_57d19654e2ae916ecd0b5c09","provider_config":{"source_label":"minimal"},"provider_mode_key":"Minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"14952d32bd1e9f4e5d4aa3b1dbae5b5924dfea982ddd8bc682e4a93d404a8ed6","metric_details":{"license":"Proprietary","variance":19.192439505514923},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3-flash","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5b053ed5719116b9fad7f22ed999c736b5a1c7e6ae572a9edee20b8bdb8276c1","result":{"confidence_high":1447.4472281151134,"confidence_low":1430.2743464665257,"sample_count":10896,"score":1438.8607872908194,"score_display":"1439","source_stated_rank":72},"run_fingerprint":"8f218679d3bb8ca4cb066469e8528d563375a08d66e28693b4a6da0b181c3b88","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f0275c6b0ee9733a34f03d3d","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"0c1438b7df5491a3100ecbf7965361850aa4b6d07a42cd3cb05a1d2f1166b0d0","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"6c9153fd9bf954f52f55708b6c204d17daa8d6134ba319bb2a6917e7a05d960e","result":{"confidence_high":1388.45,"confidence_low":1377.79,"sample_count":20911,"score":1383.12,"score_display":"1383.12","source_stated_rank":95},"run_fingerprint":"6d19b0fd5e49eefd035a0b7d015bfa7c7a3d15f89f760d8b6cff659eaf50cb55","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9998501817383d09d031e83a","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fff108de585457848e3a7308641527a479cc2342f4a7c54650d7a6a1ae6a72fe","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"c01698a40f8b0cbf99e6adf29d614a2f39760fcf6be2ba1feb28f0cfd3503b90","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.6,"score_display":"61.6","source_stated_rank":46},"run_fingerprint":"9f88c66efe233cc2cb4555c80c88659e1a8aa71b091e66012278d21b504fe483","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7572b28212257731460cd760","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":286,"at":"2025-12-23","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-23","status":"stale"},"measurement_id":"08e9c1f3097d02f3a9f3c7eb7a36eab717f19d783703890edaa93bf1df3043da","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"standard_error_points":3.0,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":"2025-12-23","source_scaffold":"Gemini CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Gemini CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"39e8868dc238642ac07c9f92879425150a9e92bb5cdaea49b99287cb3abd10d7","result":{"confidence_high":56.88,"confidence_low":45.12,"sample_count":89,"score":51.0,"score_display":"51.0","source_stated_rank":72},"run_fingerprint":"b71ab6d75c1247382cada70c731638f3337dd2643fd5afb709f5a8bca6266454","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f34a44cfeba494e3650a82e9","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":292,"at":"2025-12-17","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-17","status":"stale"},"measurement_id":"983c27fcd70fa112d1d49d52c57bfadf7bd749dced58341b7a08dd5339f13eb9","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"standard_error_points":3.0355882645,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Flash","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":"2025-12-17","source_scaffold":"Gemini CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Gemini CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"36068b95bbf935abd7b15bedbfa4c4c41bdf47de27e173754a79b18321242e1b","result":{"confidence_high":53.36548333552,"confidence_low":41.465977338680005,"sample_count":89,"score":47.4157303371,"score_display":"47.4","source_stated_rank":78},"run_fingerprint":"1b0b0bc671122bfd9c8e7a5f5ef84ed0dd9226219ff9a563b90512ca151bfd88","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f34a44cfeba494e3650a82e9","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":292,"at":"2025-12-17","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-17","status":"stale"},"measurement_id":"6dc9ef072839a5b77cad5978cdfd3b000dbae536dda827aebac8d797a41e2bd3","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"standard_error_points":2.7856468377,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Flash","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":"2025-12-17","source_scaffold":"Junie CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Junie CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"2c8be6a956a3eca4451bb4733e45481817d3b533ff9aeb38e4bb38f5fc9cbc76","result":{"confidence_high":69.729530723192,"confidence_low":58.809795119407994,"sample_count":89,"score":64.2696629213,"score_display":"64.3","source_stated_rank":37},"run_fingerprint":"adbefb224217b8c997087a993c4336e5b63ca1b2be5891eeca96158423d73738","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f34a44cfeba494e3650a82e9","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":292,"at":"2025-12-17","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-17","status":"stale"},"measurement_id":"e5fe563745035b5ae561a4265d7ef21a7e33f7df1bfcb794b32049c24b491f6f","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"standard_error_points":3.1299813473,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Gemini 3 Flash","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":"2025-12-17","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"cc96a1ab81b88b835768147304d750c3eefe61597c0498d6e17ee4aab326d806","result":{"confidence_high":57.820156699108004,"confidence_low":45.550629817692005,"sample_count":89,"score":51.685393258400005,"score_display":"51.7","source_stated_rank":70},"run_fingerprint":"345d840642b7cce2ffa3ab61669e00925d722819a62819171c677d8c211834c0","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f34a44cfeba494e3650a82e9","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f8a1a260884a199864f3105caf1c95077a2c6b19b8da2fea35c1950d2d8d8c30","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.154312,"date_is_proxy":true,"latency_seconds":376.158,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.297},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3-flash-preview"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"0dbf059c9568bd2ff562112abcd0c8ea424ccbd4e040d7ed0648b006e093bdc6","result":{"confidence_high":56.47512,"confidence_low":51.39088,"sample_count":null,"score":53.933,"score_display":"53.9","source_stated_rank":53},"run_fingerprint":"a1151a6a10051b41c0f81fbdc4e5a9dd330e16a0221e6d7aca9f62c4c59bfa2e","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_bde217160cbff9a7aeb3cce7","split_or_window":"terminus-2","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":139,"at":"2026-05-19","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:31Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-19","status":"stale"},"measurement_id":"7e3bb744a1192b1dab36e95aeb18b8fa7b727f9dc195ffa216f6173cd79d3ef6","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","evidence_sha256":"5bc317ffea95778d4ff32d6a1443305346272d8f30d548679ff6268476ab2c9b","kind":"reviewed-static-report","published_at":"2026-05-19","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","version_id":"41209cd479f50452179bfacedd2070b25afa7ef4e03e1235b8a1232d9c54bd24"},"source_locator":"Model-card benchmark table · Terminal-Bench 2.1"},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider evaluation; pass@1; default sampling","reasoning_effort":"reported","result_published_at":"2026-05-19","results_as_of":"2026-05","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"810a0c300401e897de33867304c75d2c75e569f33c773a1328d8926ebc663e29","source_published_at":"2026-05-19","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"bcb4b343d34449dcf9aaeead2c13a11446366374408351358d9bc234c9274cbf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.0,"score_display":"58.0","source_stated_rank":null},"run_fingerprint":"4ee0b8a32353bfe64800d288665266b1dc08cca0d41c5d9c800e7a9d5ea23a47","source":{"content_hash":"b8470a7cfd589a7654621e656bf5013578ef1b0d910da5c04b018a4cc61a6441","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/","id":"google-gemini-3-5-flash-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · Terminal-Bench 2.1","name":"Google DeepMind · Gemini 3.5 Flash Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fee9b688e55e7a2cfd3af24c","provider_config":{"source_id":"google-gemini-3-5-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":217,"at":"2026-03-02","basis":"evaluation_at","evaluated_at":"2026-03-02","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"532cf41c9cefb993bb34a8fb5074d88b77bf7c65b64589566bf4263b1155e857","metric_details":{"cost":null,"pass_2":15.46,"pass_3":10.31,"pass_4":7.22},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"terminal","submission_slug":"gemini-3-flash_sierra_2026-03-02","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"65cc8cfab0984dda436d298ef1a18fd2db48c8b2d5c686cddf0f37bf4300de51","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.32,"score_display":"27.3","source_stated_rank":null},"run_fingerprint":"2a3687df5ba23fddad72c655bddd47c288dd0d3ded70288efc58dc7452f022eb","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_385ff75e5783055d4ca2f505","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"e1f9e64009a75c086993ae358540a7901f1e156b108aa23731e624e6f7d05d71","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 87 scored items with a 95 % Wilson interval.","expected_trials_per_mode":261,"normalized_gain":null,"recomputed_date":null,"selected_trials":261,"skill_lift":20.3884},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"gemini-cli","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"gemini-cli","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"b364c19f09c3235a2849dcf46f5f7497a801c57e4a3db1ecf5efd428de0c8418","result":{"confidence_high":64.652245,"confidence_low":44.171721,"sample_count":87,"score":54.6068,"score_display":"54.6","source_stated_rank":6},"run_fingerprint":"f1096254a2e3b14483283fe2787066724e728caa86ac256ba3707c65b05d0b73","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_24a77b9d93725e789667e3fd","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"6d590ce625f9b569cffeb6d8b60f21796db239ff47484c4fb5df08eff861ea75","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 87 scored items with a 95 % Wilson interval.","expected_trials_per_mode":261,"normalized_gain":null,"recomputed_date":null,"selected_trials":261,"skill_lift":20.3884},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"gemini-cli","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"gemini-cli","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"8c3496e82aac91ec0feee659c4f06e8ed97c98e977dbb8aefe3da75bd141f00a","result":{"confidence_high":44.665126,"confidence_low":25.10645,"sample_count":87,"score":34.2184,"score_display":"34.2","source_stated_rank":8},"run_fingerprint":"51f68710b8093a844ab75a28bb6af7aa48d0d9b9a1a337d826085841b9f7c064","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_24a77b9d93725e789667e3fd","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"75e2b7713162cd2e4f4974225bc0e17ee703c17ba44b9c515971af98bfc6e1d6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.395","mean_standard_error":"3.3","model_release_date":"2025-12-17","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.3,"upstream_source_url":null},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"6f47bbd313086b1bd1ea8eeda64b4b997e81d436ca2a91f2d082308e388f83f5","result":{"confidence_high":30.468,"confidence_low":17.532,"sample_count":480,"score":24.0,"score_display":"24.0","source_stated_rank":31},"run_fingerprint":"154d07fbd75e8e8489a5c095935ce73e2194776ec5baeaa42047d8f7e73eb3e8","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3259dc57ae6944bc60519ed7","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ab369fdde7df27d5db71a866b5199cc9c4f1e7160dcaa02cffe65b1f4d52b1b1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-12-17","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-flash","name":"Gemini 3 Flash","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3 Flash","source_effort":null,"source_model_version":"gemini-3-flash-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"b62090d365143d132738ff4217234b598cb3a10704b5d2fbccbb260b57b3516b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3634.715999999999,"score_display":"3634.72","source_stated_rank":41},"run_fingerprint":"c6f197f981c37382515bd9b11b40efdb3b1e64deb21855e5759da1dab3919903","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bc2d3b57176164ba1eac2f22","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"145c3ce898d6190811003a6fae5cb4229967c968a57df6fe893b77eb5e5fe44e","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-06-16"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GLM-5.2; model release: 2026-06-16","source_accessibility":"Open weights (unrestricted)","source_model_name":"GLM-5.2","source_model_release_date":"2026-06-16","source_model_versions":["glm-5.2_high","glm-5.2_max","glm-5.2_medium","glm-5.2_none","glm-5.2_unknown"],"source_reasoning_efforts":["off","medium","high","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"73873156ba6042cadd66266adb01f56c2868bf807beb4545c637e04313d467b5","result":{"confidence_high":153.95,"confidence_low":149.83,"sample_count":null,"score":151.78,"score_display":"151.78","source_stated_rank":45},"run_fingerprint":"9b8d812da893bc4ce863bc3cb3024884575e40df575265beaf8bef3b5d79e06d","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_04566da2338e5d8f2694eef1","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":22,"at":"2026-09-12T16:29:30Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-12T16:29:30Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b681437e16c79135254397251905e24b8a53d63cc9a2eb6c705eeb55a3496a55","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (max) · intelligenceIndex"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-2","upstream_model_label":"GLM-5.2 (max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"36dd1bb8fe0213cc41cec022a6df1441bdad26ddfc4393c823c958212551d656","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.005232,"score_display":"34.0","source_stated_rank":null},"run_fingerprint":"562b047259e564f1fbea14c38d2021e2b095227acbfba284237e3f2247734ac7","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0c605479afa0f3be0629138774a08a8db8ea2f35d3d921e67b7a83760278ed11","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · intelligenceIndex"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"glm-5-2-non-reasoning","upstream_model_label":"GLM-5.2 (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"8522fe32aea5255b02d38fd537be625a2af635c20dc3e34d393f2bf6367bc010","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":22.432347,"score_display":"22.4","source_stated_rank":null},"run_fingerprint":"951822669d868d12322512fdf955e8d69e7a0d813401c03f0800688f7a6a399a","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"177deec0f128a04f5be52eac76316f10548eed1296a768d563e0dedce540fa3a","metric_details":{"license":"MIT","variance":4.813812205075839},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1480.0697218323157,"confidence_low":1471.4692424604775,"sample_count":45399,"score":1475.7694821463967,"score_display":"1476","source_stated_rank":32},"run_fingerprint":"70e3d281de6bbe3c35472e7c6f187aed718147ef330df140831561fca22f2944","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"02369d341650d2eb3af48afb7960c22138a01451937363529ce92205e65be545","metric_details":{"category_scores":{"Agentic Coding":51.76766666666666,"Coding":79.654,"Data Analysis":73.73966666666666,"IF":62.29175000000001,"Language":76.24166666666666,"Mathematics":89.78125,"Reasoning":78.625}},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.15728571428572,"score_display":"73.16","source_stated_rank":46},"run_fingerprint":"5e3d12a273cc6af9f2049fb26ea9f3ab9b4ff8495268c5286b83d7daf943c59b","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_17119771e437727ff7cadce7","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"34c02cf3584468e9a5bcfc8f0209e59a43a0c4ec06080a21af7c769faa2824fa","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":null,"num_parameters":753329940480,"pull_request":null,"result_file_url":"https://huggingface.co/zai-org/GLM-5.2/resolve/main/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/zai-org/GLM-5.2"}},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.5,"score_display":"40.5","source_stated_rank":27},"run_fingerprint":"92143972463ebfbff547671d533e4a8e938649b92907636430a91628fc71ee36","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c54b095bc209fa08007ec46c","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"evaluation_at","evaluated_at":"2026-06-16","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"9917395edba09f7e3719e1390b600be1b77886c024f27ca75e5d13ca1ba236fe","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/zai-glm-5-2","source_model":"GLM-5.2"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0ec13e7db5b22c21e47f8dafa1225f705e8a4159831eafb1bd4d2621d7a533f6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.5,"score_display":"40.5","source_stated_rank":12},"run_fingerprint":"a6f4c561ca57e3dad169a7fd9cefb04f56bb94c194bdb89cc801a0698b577c20","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b88287adb06f58d91aeb5d22","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b8a5ee6f7092ac23a7503082b465a949be852bd2b0b727e3fd5a68580d8da5a0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":"With tools","num_parameters":753329940480,"pull_request":null,"result_file_url":"https://huggingface.co/zai-org/GLM-5.2/resolve/main/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/zai-org/GLM-5.2"}},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.7,"score_display":"54.7","source_stated_rank":7},"run_fingerprint":"93b39b923820fb3acff2a986aaa5bb7f02e2853247f7bc66c45c86bb571bf0e7","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c54b095bc209fa08007ec46c","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e6d412214db31ebd4fddc1a9a914a4f1531ee03c8b73a6d287d13ab6ca23696b","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (max) · hle"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-2","upstream_model_label":"GLM-5.2 (max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"92ed48917c513acfb6e6762c693dc06dddf60cab9a314f607c515ab653f67106","result":{"confidence_high":43.23944,"confidence_low":39.090439,"sample_count":2158,"score":41.149212,"score_display":"41.1","source_stated_rank":null},"run_fingerprint":"f3c57afdc7d1b2a7aacb0be4cf36e8d39f6caa48d960c2e42e64384a4b2c0845","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9fb0da723fc901bf24e9f71f3b08acdd8bb6c2276798231d31ab10fd0088ab2d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · hle"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"glm-5-2-non-reasoning","upstream_model_label":"GLM-5.2 (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"de5ceb627e84c0496ab3a7016bf51beee60cc8ca4b2d93bda4150ce36c89ab03","result":{"confidence_high":11.103122,"confidence_low":8.594973,"sample_count":2158,"score":9.777572,"score_display":"9.8","source_stated_rank":null},"run_fingerprint":"5312e24cb29b26159838b9b6aa3581bd9624e912068ea0ce375ec620aef516c2","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:36:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:36:05.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dcdcedaca2fbbd70bc452dc784f0d231ee7cdc68b2a8d8003e5f2c49dd8ec46d","metric_details":{"best_score_across_scorers":"0.342","model_release_date":"2026-06-16","stderr":1.5008706182121803},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"WgjFZzojpiv54XqUhvfuUG","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWgjFZzojpiv54XqUhvfuUG.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/WgjFZzojpiv54XqUhvfuUG.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"793a7e19bb4532d6fd827caa77a0e7c75b6fe3c7661e7cb6dc360e76b8ab1b4f","result":{"confidence_high":37.14170641169588,"confidence_low":31.25829358830413,"sample_count":1000,"score":34.2,"score_display":"34.2","source_stated_rank":54},"run_fingerprint":"119336aaae908cb7675771f85662d5f3b99c4d142c822870c9e82cf9f83a078b","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aff645a01e211aea17a85a0f9d85a3fe06a1d037d0ca71b764ed3e72014527aa","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.05711,"date_is_proxy":true,"latency_seconds":224.664,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.774},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":83.333,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":null,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"zai/glm-5.2","zero_shot_accuracy":87.879},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"5e2e3f8c717da80a75423a1302c231d94146ae7d2b62ffa4df742dd3b6736b89","result":{"confidence_high":89.08304,"confidence_low":82.12895999999999,"sample_count":396,"score":85.606,"score_display":"85.61","source_stated_rank":46},"run_fingerprint":"38df59ef988e2f8917156251374e3bbfd92abe6b1296591234e8058257544d50","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_43b1a4c9d140c7bbdd7604dc","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dc1672441b980cb06d026e398acba19fb2b6d6b396a55128368569305f9efe94","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (max) · gpqa"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-2","upstream_model_label":"GLM-5.2 (max)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"34c3a6bf82c2887007a3c8e6e83dcadba70032b4b97263f7baec1be6a0eeb278","result":{"confidence_high":93.039607,"confidence_low":84.446897,"sample_count":198,"score":89.494949,"score_display":"89.5","source_stated_rank":null},"run_fingerprint":"f06c10beecc5c4fda174e65902648d9017732c01413cff3b4f2ef36ab4733c9b","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (max) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b8df6a2515c9018507a41b7ebed3cd04b82cab4d0125d1be6adc06e204684c11","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · gpqa"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"glm-5-2-non-reasoning","upstream_model_label":"GLM-5.2 (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5beab610714a9f5f7483f53e076930f86ce89d54426fbdc7692327cf394cb70e","result":{"confidence_high":74.645577,"confidence_low":61.818662,"sample_count":198,"score":68.585859,"score_display":"68.6","source_stated_rank":null},"run_fingerprint":"89adb623b6c9bd25f679f3e113cc59549c3fa8f213a2ffe3d42a38b523f23520","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":102,"at":"2026-06-24T20:55:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-24T20:55:39.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"cf6b95449f60951d359f17666c3ef3bc377ee91ce5005722b312d82cbed90afa","metric_details":{"best_score_across_scorers":"0.9185606060606061","model_release_date":"2026-06-16","stderr":1.6123207533687733},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"c5kpB4axzmygRywWqBKFmv","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"b1bcf6a629f799191e3b86e54568180a1e15105086f7795c14909666e82e3785","result":{"confidence_high":95.01620928266341,"confidence_low":88.69591192945781,"sample_count":null,"score":91.85606060606061,"score_display":"91.9","source_stated_rank":28},"run_fingerprint":"cbc78b1018d4dbd5fa3ec24611f8582c8fc8d6b51cde390fc0d51ac2e6c26077","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":55,"at":"2026-08-10T17:45:10.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:45:10.000Z","first_observed_at":"2026-08-28T00:52:16Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"34714c9fb1932e024247c78b0adefaeb3f8e787ef815ffbb5b0e882bfe0e87ec","metric_details":{"best_score_across_scorers":"0.8787878787878788","model_release_date":"2026-06-16","stderr":2.325315795194205},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"SB6Qj6rCBDk8jZmFcCTzWc","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSB6Qj6rCBDk8jZmFcCTzWc.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/SB6Qj6rCBDk8jZmFcCTzWc.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"998d083bdb7cfe53d7946cc01045c3812fa8d2f3bdf8f01471d461be29404e92","result":{"confidence_high":92.43640683736852,"confidence_low":83.32116892020723,"sample_count":null,"score":87.87878787878788,"score_display":"87.9","source_stated_rank":63},"run_fingerprint":"65aed823562858bcfb82e9b10b833e3ddbdd2c557c4d19e7d801589e6fd9cab3","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_dc44779e42b6ce301d315046","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":55,"at":"2026-08-10T17:45:32.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:45:32.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e3d1f996eef9cc6fc252825a7f8db863c1b45a3a6edcd0e3396f39df4cf5d2a8","metric_details":{"best_score_across_scorers":"0.7121212121212122","model_release_date":"2026-06-16","stderr":3.225883512300997},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"YDQJAMgMjLaikPkeVDbAjr","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYDQJAMgMjLaikPkeVDbAjr.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/YDQJAMgMjLaikPkeVDbAjr.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"ecca03f4e2061b9996bcde67bcc490e1b91979217590d853c3e48dfcbc1fbbc2","result":{"confidence_high":77.53485289623117,"confidence_low":64.88938952801126,"sample_count":null,"score":71.21212121212122,"score_display":"71.2","source_stated_rank":177},"run_fingerprint":"4e6095cf6832e465f7b0966a7f2b0c8f27c6ea736d22f5a220a1fb63815d8f75","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bd9bd20f337235f3110f46b25acc87b8473ee1d4bc3117dc53f30dfe41a9942c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.013838,"date_is_proxy":true,"latency_seconds":51.64,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.334},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":null,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"zai/glm-5.2"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f332c2507af1b34ab029f419d27822241f59c8ba5942118e83e03344bb877234","result":{"confidence_high":87.36864,"confidence_low":86.05936,"sample_count":null,"score":86.714,"score_display":"86.7","source_stated_rank":39},"run_fingerprint":"01af5a2b180f527da0d75cb797031c5fa97b3d19f40147dd3fdd6bfe066a6562","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_21a68f413995c1824054e5f0","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25T00:15:08.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-25T00:15:08.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a1d859101c4d4f2cbc51cf17a42924e9f690ffa48ff5bc62751d309ab6cd61ce","metric_details":{"best_score_across_scorers":"0.8638888888888889","model_release_date":"2026-06-16","stderr":4.2197793305105735},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"XjFqou6MyT2XLpkUsCrgxM","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"15d072ac1920c685176ffc0de7e75f477ea07dbe82bf73801bbd24efbc815915","result":{"confidence_high":94.65965637668961,"confidence_low":78.11812140108816,"sample_count":45,"score":86.38888888888889,"score_display":"86.4","source_stated_rank":89},"run_fingerprint":"5162fc5563a689ec39f359c62e5382fbeddf9b9dd9f3d6a8c365603d1acbdcba","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:45:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:45:24.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dcc8ff8f8c3a492352763c6d4a7601efa65ccd7c87ec2af4809970d59ef66a3a","metric_details":{"best_score_across_scorers":"0.7555555555555555","model_release_date":"2026-06-16","stderr":6.478835438717},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"9uKmMPn6eiRPsfcLKod7M7","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9uKmMPn6eiRPsfcLKod7M7.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/9uKmMPn6eiRPsfcLKod7M7.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"90fb814dc6cb217e3673aea18a1175323fc6f6946bbc11ebf1f8331e6668c68b","result":{"confidence_high":88.25407301544088,"confidence_low":62.857038095670234,"sample_count":45,"score":75.55555555555556,"score_display":"75.6","source_stated_rank":123},"run_fingerprint":"ed0872f017d45553e8f9e83383114d9cd961bb827e1b1b9524bd92705dee8307","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_dc44779e42b6ce301d315046","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:45:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:45:48.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0b30b8b2ebc1dd3ec00ae0ab602ce6b867880eb20c41a1819a7d63b193b97b18","metric_details":{"best_score_across_scorers":"0.28888888888888886","model_release_date":"2026-06-16","stderr":6.832943242540508},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"J8hYKbXiGhetQRwu5bUSp6","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJ8hYKbXiGhetQRwu5bUSp6.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/J8hYKbXiGhetQRwu5bUSp6.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e3d5243d25ab4dbb5b94aa08e59f9788878aaf5efd7d8eb7c6305a0280e0f3b4","result":{"confidence_high":42.28145764426828,"confidence_low":15.49632013350949,"sample_count":45,"score":28.888888888888886,"score_display":"28.9","source_stated_rank":221},"run_fingerprint":"3e3685b1b131d72b496de100e3bd42f200b6090d8307c12f1d6f5fc908ad75c8","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":107,"at":"2026-06-19T16:13:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-19T16:13:05.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"34097e1a743646b396595949825623224240eff864aefad11128fa091ff5af66","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.592057761732852","leaderboard_private_problem_count":285,"model_release_date":"2026-06-16","public_example_count":10,"stderr":2.958195251960623},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"TTuCPacMDpNaCSs3Hpbdjp","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GLM-5.2","thinking":null,"upstream_model_id":"glm-5.2_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"2b719d2dc1d51447164fab6e154bf5117436d13b91c8542fa3543c966796cda1","result":{"confidence_high":65.00383886712801,"confidence_low":53.40771347944237,"sample_count":285,"score":59.205776173285194,"score_display":"59.2","source_stated_rank":36},"run_fingerprint":"31652c431681a99cb89be9f3843b7c9f59d2a12c3ce37581991ce72c4b61d64a","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-29T11:07:19.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-29T11:07:19.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5351dfcae37799f42707310dbfbd7970be39aff217add8bd78fb727eb5bb3e1a","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5473684210526316","leaderboard_private_problem_count":285,"model_release_date":"2026-06-16","public_example_count":10,"stderr":2.95360982699221},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"UnKHyc2idtTmoTUGxGeNJV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUnKHyc2idtTmoTUGxGeNJV.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/UnKHyc2idtTmoTUGxGeNJV.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GLM-5.2","thinking":null,"upstream_model_id":"glm-5.2_low","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"633b9f6843376f40b6d4cbc06f57fa101fa0c62670a4b52528c4873c386772e3","result":{"confidence_high":60.525917366167896,"confidence_low":48.94776684435843,"sample_count":285,"score":54.736842105263165,"score_display":"54.7","source_stated_rank":44},"run_fingerprint":"1cb33b05b66c982b2810e3732da410ac3900412b8bfc7eb9d68283433ac0168a","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_dc44779e42b6ce301d315046","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-29T11:07:59.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-29T11:07:59.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"756dd4f6f062b6bc50575fea531037de443244de1b210ff83636590801b7aef7","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.4245614035087719","leaderboard_private_problem_count":285,"model_release_date":"2026-06-16","public_example_count":10,"stderr":2.9329899789943816},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"ECdG3m8n5ABfWVZ55rPFmT","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FECdG3m8n5ABfWVZ55rPFmT.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ECdG3m8n5ABfWVZ55rPFmT.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GLM-5.2","thinking":null,"upstream_model_id":"glm-5.2_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"e2f58152bbe0ed76d83ef6b0b0791bfa421eb67ba3d4e9adebf71925c36b350b","result":{"confidence_high":48.20480070970618,"confidence_low":36.707479992048206,"sample_count":285,"score":42.45614035087719,"score_display":"42.5","source_stated_rank":55},"run_fingerprint":"368237573d14f715500c00e9ba8a66e75f2890f0a23dbcacbbe45d23c7f1a410","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f16aaff082bc46810e1390aedf689b22128f31d962fc8d16a59d1f2c6ae47bb0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.2501,"model_release_date":"2026-06-13T00:00:00.000Z","results_url":""},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"","model_type":"CoT","upstream_model_id":"glm-5.2"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ee7689cab2ecdae3f0d03646646ae70a43a48a3d5e8191ac81a5dfd541badceb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":22.775000000000002,"score_display":"23","source_stated_rank":128},"run_fingerprint":"da10af33836a3d7ffe1dbbecc342f748886529d60363df8d190928609cce99f5","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_35d0fa8752189fb3eac4cea6","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4ff76f7a69c43329610ec40e565ca511fa937134909b14921392b1ae6b42ed3e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.2501","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.2","source_effort":null,"source_model_version":"glm-5.2_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"776006862bb056e98126be23072604d679909a18c082535ea501f41041be78cb","result":{"confidence_high":27.381343,"confidence_low":18.743564,"sample_count":360,"score":22.775000000000002,"score_display":"22.8","source_stated_rank":124},"run_fingerprint":"5339016b4868d4e24468c4cbd3f280080960afc3777e3710e115e96187974f65","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9686f231d5fd973e7be1ac20","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":107,"at":"2026-06-19T16:13:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-19T16:13:05.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"2094e89d5f23bc47dcf155484ab2a1cbaddd3e3ddb751211b544b3cff2137b26","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.2926829268292683","leaderboard_private_problem_count":41,"model_release_date":"2026-06-16","public_example_count":2,"stderr":7.194088392074452},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"JebA8dYLVov6H49ZQSLXh5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJebA8dYLVov6H49ZQSLXh5.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/JebA8dYLVov6H49ZQSLXh5.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GLM-5.2","thinking":null,"upstream_model_id":"glm-5.2_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"908684107da51a0d817af2b4d3f9faf3d656f68c7600d9fae11ffff479060d82","result":{"confidence_high":43.36870593139275,"confidence_low":15.167879434460902,"sample_count":41,"score":29.268292682926827,"score_display":"29.3","source_stated_rank":34},"run_fingerprint":"97957a54410e82f2f41eefbfd201936e870d5cf5bb312c2c0b32960ce0540c2e","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5277c9b0bce3b8b2007696461a02947419cd7a02cb19b874fc67adc280bbe447","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.2 (max)","source_effort":null,"source_model_version":"glm-5.2_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"be31bd37ae61b1d561478ee1382318c80c3696844148ce5360b8fd1e088b0ea3","result":{"confidence_high":31.678669,"confidence_low":13.027407,"sample_count":71,"score":20.8571428571429,"score_display":"20.9","source_stated_rank":48},"run_fingerprint":"ed8a2aa3ea6d01f5e2edac0935a8ed0886502b6d72b5079c8b31b4ae4f96fc70","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c54a1972c3201d0054e41d6afbac24d0e2debfe0821818c4b142f1d43ebb0e61","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.2 (Non-reasoning)","source_effort":null,"source_model_version":"glm-5.2_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6138b15acd4a66cb1dcaa10f715ce9a01fee5fcb18d2a5afbed185b38d7c219f","result":{"confidence_high":10.175118,"confidence_low":0.920927,"sample_count":71,"score":3.14285714285714,"score_display":"3.1","source_stated_rank":121},"run_fingerprint":"b798926012d573d4e819765723476e3f2776316a70bdade8e4847993f0cf8f09","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6c9351175d60fb89ecded17cc47d03a87830cbdd0af002fb0d5a8a0b537a2f1c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (max) · critpt"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-2","upstream_model_label":"GLM-5.2 (max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"d7b2acd48990466fd85d52e38e3f1e32db1749e396cf9a715702bd8a07df66eb","result":{"confidence_high":31.763497,"confidence_low":12.983096,"sample_count":70,"score":20.857143,"score_display":"20.9","source_stated_rank":null},"run_fingerprint":"0708a3e79a992fc9a0da1de75d09845f8c8d037471c06da0ba864198572c8aef","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"99304dc81280cd0d43118209cfc25dc5fbd960bf38f5d08132f193a544bd947f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · critpt"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"glm-5-2-non-reasoning","upstream_model_label":"GLM-5.2 (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6733dd944937d3734e149049ef82239937aa7f11a98fc45c64e970f14f7280d9","result":{"confidence_high":10.24743,"confidence_low":0.913758,"sample_count":70,"score":3.142857,"score_display":"3.1","source_stated_rank":null},"run_fingerprint":"3f522aa9072eb8444e3186c359ad895912dc60fbc8fd1c113e0e5067bbb7ae4f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"06170a1a0eaf528902cc74463406863daf90ff22b09596ab15d8c1b76f7a3911","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.2_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"fbbd9e0d0e3dae62501f10feb8ae007290e95726caad1c8211fbd34936a787a9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.8,"score_display":"58.8","source_stated_rank":34},"run_fingerprint":"98a7cab7257b194d8a755defc7db20b288ce4e79428b4ee22571a0fec1295316","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c362d9b546b44a292da204a3","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":109,"at":"2026-06-17T23:43:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-17T23:43:15.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"422d12f68d5d96c20ee4706cabb5bd4760d11312bcea211986031aa117180857","metric_details":{"best_score_across_scorers":"0.21","model_release_date":"2026-06-16","stderr":4.093601807403323},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"G8hwpomZQaMniWH5XUzTRe","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"87e17c36e98a11d679c496f42691802b804857b1408e073f366337f95444f3c1","result":{"confidence_high":29.023459542510516,"confidence_low":12.976540457489486,"sample_count":100,"score":21.0,"score_display":"21.0","source_stated_rank":80},"run_fingerprint":"731e3c16e76cf2cde9a0194ff8b9dce0ba05a0bedf11f79c9bce5d1dbdd52f3a","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:45:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:45:14.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"74477cbae7755ea4a282f5dce3537a96a035624a79b7882c2dc23c5ea167fbdd","metric_details":{"best_score_across_scorers":"0.14","model_release_date":"2026-06-16","stderr":3.4873508801977695},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FUE7jUup3RsXiXpYTjMKb5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFUE7jUup3RsXiXpYTjMKb5.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/FUE7jUup3RsXiXpYTjMKb5.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"41de41892ee66fdbb9aaccfb21bea9d7d523c3add619155d9d64701e25f4b055","result":{"confidence_high":20.83520772518763,"confidence_low":7.164792274812374,"sample_count":100,"score":14.000000000000002,"score_display":"14.0","source_stated_rank":113},"run_fingerprint":"4aadea0668849f21465e907bdcc729bff005a27242eae096c822598e2c82a62e","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_dc44779e42b6ce301d315046","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:45:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:45:54.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c093f9b6492fdae20e0f9d25b078b49bd5813137abfd135e6fa04dcbae64834e","metric_details":{"best_score_across_scorers":"0.06","model_release_date":"2026-06-16","stderr":2.3868325657594203},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"kwT2PhMLMheWkxaqwh8PKq","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkwT2PhMLMheWkxaqwh8PKq.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/kwT2PhMLMheWkxaqwh8PKq.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"874f672990ed6e52ccc664c87b814845e016ed85906bd80d26061d46fcda1443","result":{"confidence_high":10.678191828888464,"confidence_low":1.3218081711115364,"sample_count":100,"score":6.0,"score_display":"6.0","source_stated_rank":155},"run_fingerprint":"ac6f880ee019e65e73787b6a37707ab1e2cf1e23ada7ecb352cb4bb47d458c4a","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"43f5c8ec01eacc7d1d4c3492e8124528525645c937684218c68d204fecb04121","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.1916","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.2","source_effort":null,"source_model_version":"glm-5.2_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0bc797d563c9920ad5117d503a938d125f8b22dab8024cd5dcc36cb2971a2a71","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.0,"score_display":"77.0","source_stated_rank":117},"run_fingerprint":"9e2b38906198de1eeea076b5176ba3d9f9b234a61251c0169e236b17b840e30b","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fbde99d386073827dc60739c","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":110,"at":"2026-06-17","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-17","status":"stale"},"measurement_id":"b8fe8f13301eca0dfaa4dd7971811fa7dc3d8e69eaf7d67d38ce15e126d0a3c7","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run; vendor scores using a different agent or tool setup remain separate supplemental runs.","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","cost_per_test_usd":0.710002,"date_is_proxy":false,"latency_seconds":669.667,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/zai_glm-5.2","source_locator":"Results · Overall accuracy","stderr":1.689},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":131072,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"c248e3defac0e971fdf6ed1bc878fea5f152877661b4719c38c496230174eb05","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.2"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"ebca2b6fb976e93c57d1b0c53f56f4a61ebc845f67fb5a067c6c5c3554abe5f4","result":{"confidence_high":86.11044,"confidence_low":79.48956,"sample_count":500,"score":82.8,"score_display":"82.8","source_stated_rank":18},"run_fingerprint":"4c702051400591ec4d467cc763d6ee46a573cb0932ade6a1ce88b9b863f3ec42","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":102,"at":"2026-06-25T13:13:06.902Z","basis":"evaluation_started_at","evaluated_at":"2026-06-25T13:13:06.902Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8656153b431b21dbdc85220bd7918bd181670eeb41817b0ebba88495aa359114","metric_details":{"best_score_across_scorers":"0.787","model_release_date":"2026-06-16","stderr":1.8656911901656907},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4AfhcmYVNrw6gM5u8CQsyy","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4AfhcmYVNrw6gM5u8CQsyy.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4AfhcmYVNrw6gM5u8CQsyy.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"555e48d4cd9ccddf2fb441d97ee93cb077f8620ab1f1eb6c38ae91c4a7d661c7","result":{"confidence_high":82.35675473272475,"confidence_low":75.04324526727525,"sample_count":484,"score":78.7,"score_display":"78.7","source_stated_rank":5},"run_fingerprint":"88af911c58c614f1e905fd2ad3f5a32ce6b6f0fd5d012833fa9e9dbd297aca27","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a0a57cf3d28bcd91bd306d1abf514f7107f0486cd56a6c7d30c085709d1eb8b3","metric_details":{"model_release_date":"2026-05-16","pass_at_5":81.08108108108108,"potential_contamination":true,"sem":1.1883699061507138},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"contamination-risk","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"GLM-5.2 [high]__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"ef3c892e6111a7703e93a922e80e35358681f631e166e5fcacaba61b8a3f3d38","result":{"confidence_high":65.21208789893828,"confidence_low":60.553677866827485,"sample_count":null,"score":62.88288288288288,"score_display":"62.88","source_stated_rank":4},"run_fingerprint":"2c185be4b4fa421c03cd92083933a8ef1b134e9238eac4d25873241b33bf5bdf","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0a11b05baf509af0577d53c4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"contamination-risk","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_8d9de222c7fa49c448653fc0","split_or_window":"2026-06-01/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-29T23:53:02Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ea033704eceb2357e6eee97c93f2177216325f4cf5d9989d85b1543595587994","metric_details":{"comparison_warning":"Official post-release SWE-rebench window; use only for same-window comparisons and never mix it into the canonical rolling-window rank.","comparison_warning_code":"swe-rebench-post-release-window","model_release_date":"2026-05-16","pass_at_5":76.19047619047619,"potential_contamination":false,"sem":1.934294858246657},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-post-release-window","comparison_warning_code":"swe-rebench-post-release-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","supplements_range_key":"1778803200000:1782864000000","upstream_model_id":"GLM-5.2 [high]__tools","window_from":"2026-06-01","window_status":"supplemental-post-release","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"1fb1e70500a875cd246d716d74f2891f497e4167a1987d26e3411328082014a9","result":{"confidence_high":61.88645601740155,"confidence_low":54.304020173074655,"sample_count":null,"score":58.0952380952381,"score_display":"58.10","source_stated_rank":null},"run_fingerprint":"e9475e09a9470f0249bd043d3a1d412887eae00a2476e20d4bf5bb011bb4087f","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0a11b05baf509af0577d53c4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-post-release-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"e07424058e54e99848dbd34aa3149708cc2ddec67c0f70687d75563f39da3deb","metric_details":{"ci_attempted":450,"ci_half_points":1.725629620580811,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":197,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":129.12666666666667,"mean_cache_tokens":12344954.453333333,"mean_cost_usd":3.9199075743111114,"mean_duration_seconds":2629.2083211311115,"mean_input_tokens":12606560.215555556,"mean_output_tokens":78175.30666666667,"median_agent_steps":123.0,"median_cost_usd":3.4637479200000003,"median_duration_seconds":2304.8350069999997,"median_input_tokens":11208642.0,"median_output_tokens":75760.0,"median_output_tokens_to_pass":75146.0,"median_peak_context_tokens":142129.0,"n_attempted":450,"n_passed":197,"n_tasks_attempted":113,"n_tasks_passed_any":87,"pass_at_4_points":76.99115044247787,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_glm_5_2_max","source_model_version":"glm-5-2","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"7c64f1205ab13b5ff20787d49fd0c8b5dda15db5f0285f7bad596dd073bcb96a","result":{"confidence_high":45.50340739835859,"confidence_low":42.05214815719697,"sample_count":450,"score":43.77777777777778,"score_display":"43.8","source_stated_rank":56},"run_fingerprint":"53857893dabf0f8bdf12a0aa54e6e5ff0538ee519f53155c070c28c8759178e1","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"f58e9413b4454de6786ab0193b0e2ac6fdafb8966079c8ee75643efb3e28f99b","metric_details":{"ci_attempted":452,"ci_half_points":4.750072947921655,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":164,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":121.88274336283186,"mean_cache_tokens":8861413.805309735,"mean_cost_usd":2.8355164725663715,"mean_duration_seconds":1794.4733617809734,"mean_input_tokens":9070605.707964601,"mean_output_tokens":54245.50442477876,"median_agent_steps":112.0,"median_cost_usd":2.25012998,"median_duration_seconds":1565.2561315,"median_input_tokens":7049445.5,"median_output_tokens":48956.5,"median_output_tokens_to_pass":47433.5,"median_peak_context_tokens":102096.0,"n_attempted":452,"n_passed":164,"n_tasks_attempted":113,"n_tasks_passed_any":77,"pass_at_4_points":68.14159292035397,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_glm_5_2_high","source_model_version":"glm-5-2","source_reasoning_effort":"high","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"d73b40ca263f3bb357829f6c4ecc728d22ef8fc1742b6561d4e4a56ec18f4b2a","result":{"confidence_high":41.033258788629624,"confidence_low":31.533112892786306,"sample_count":452,"score":36.283185840707965,"score_display":"36.3","source_stated_rank":60},"run_fingerprint":"51dacf0955f961613651ea3acf0d4db8b01afde48d7e4ee50aca7b88033d2558","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0a11b05baf509af0577d53c4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ad4b33779069bae16612ca13b859bba44c136727c8f91b5fed3ff4ee0bae2064","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"129.12666666666667","mean_cost_usd":"3.9199075743111114","mean_output_tokens":"78175.30666666667","model_release_date":"2026-06-16","notes":null,"pass_4":"0.7699115044247787","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5-2 (max)","source_effort":"max","source_model_version":"glm-5.2_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"a9c7078b7497cd695c992dfd3333f2db5693bc845cda8fd52eee47bde28277f1","result":{"confidence_high":45.5034073983586,"confidence_low":42.052148157196974,"sample_count":113,"score":43.777777777777786,"score_display":"43.8","source_stated_rank":54},"run_fingerprint":"cc6da3bd86347f5a51347c9d89325c05a3544ddfbc4b86d8bbc646a63bd1d927","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b87818ae929b34440e2398d666093e1d48a61e798a6e15a1aeee52af8e7800ab","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"121.88274336283186","mean_cost_usd":"2.8355164725663715","mean_output_tokens":"54245.50442477876","model_release_date":"2026-06-16","notes":null,"pass_4":"0.6814159292035398","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5-2 (high)","source_effort":"high","source_model_version":"glm-5.2_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"21321bceb38ba0088e08179f2c1bc301b01ebd7ba8e62f67ebca286a035da4de","result":{"confidence_high":41.033258788629624,"confidence_low":31.53311289278631,"sample_count":113,"score":36.283185840707965,"score_display":"36.3","source_stated_rank":59},"run_fingerprint":"6fc3dab1d1841f3df92c08d8917c5448f78007677c216042257be5e8ea0b8630","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0a11b05baf509af0577d53c4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_aa29eb18bdb9e43cee539f4d","split_or_window":"zai-openhands-tailored-prompt","unit":"percent","version":"SWE-bench Pro · Z.ai OpenHands vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:19Z","last_confirmed_at":"2026-09-16T06:32:03Z","observed_at":"2026-09-16T06:32:03Z","source_freshness":"expired","source_observation_age_hours":463.9,"source_observation_status":"overdue","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"e15bf42579feb2e7f2065c69d15ce40b07b88ff0416b5694ad416b60dabeba1d","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Launch benchmark table · Coding · SWE-bench Pro · GLM-5.2 column; Evaluation details · SWE-Bench Pro"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Z.ai","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenHands with a tailored instruction prompt; temperature 1; top_p 1; max_new_tokens 32k; 400K context; exact task revision, task count, run count and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-06-16","source_content_hash_method":"raw_bytes","source_content_sha256":"a9e8c2b6f34717d69e3a0aa26bb117256a4d8c95bd299910c2a693000ee88fe8","source_published_at":"2026-06-16","tools":"OpenHands coding-agent tools"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"OpenHands coding-agent tools"},"protocol_fingerprint":"99fc4cef8010e0643d87d80ddb87946b7222f6c359dec21c878cd22d0e12a155","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.1,"score_display":"62.1","source_stated_rank":null},"run_fingerprint":"a5dc302f05d6c07d2c2d854a71a2c0a3c56327161824615189a20b3ce839b3d9","source":{"content_hash":"2b25311712ee2dd95319406ca612fd13303e81cf3f2819759a111412e6dec532","fetched_at":"2026-09-16T06:32:03Z","first_fetched_at":"2026-09-02T12:19:34Z","homepage_url":"https://z.ai/blog/glm-5.2","id":"zai-glm-5-2-report","last_fetched_at":"2026-09-16T06:32:03Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Launch benchmark table · Coding · SWE-bench Pro · GLM-5.2 column; Evaluation details · SWE-Bench Pro","name":"Z.ai · GLM-5.2 launch report","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://z.ai/blog/glm-5.2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a3656fede2637c7d31099b3a","provider_config":{"source_id":"zai-glm-5-2-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"805c9209e35ca92976fd1c0aacd8f2b27a12241471a88be0551a89b93edbbd0e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.76","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-06-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"58","tokens_per_task":"35946","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM 5.2 Max","source_effort":"Max","source_model_version":"glm-5.2_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"a805e490f7b8ed5fb32d8d58da4cf2d99066dff4fa611224ce387b61c3e9509d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.00000000000001,"score_display":"55.0","source_stated_rank":48},"run_fingerprint":"5df306ab3f83bc4aaaff7d8c6e0d39e9d55aebb5b06234dd21647e201aac5c6d","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"639fba003a01f48402ba56f7d68e531fc8e01af815391ff4d27b593f42961adf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.19","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-06-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"49","tokens_per_task":"21829","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM 5.2 High","source_effort":"High","source_model_version":"glm-5.2_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"0f7358fd650ef6b1d8734031f65d0cc07f9e506462e9522d4df49d24dadf58e9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.5,"score_display":"51.5","source_stated_rank":57},"run_fingerprint":"9bdb7c1856ce6aedf936fa8a1adcf9c5282e062cb492af0616b4284d5d0987e8","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0a11b05baf509af0577d53c4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3433ed385fa528d7d674a7e91d59360a173b4e549bb80cdc99f1fcacd200cf6e","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM 5.2","source_effort":"none","source_model_version":"glm-5.2_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"6c9a24d40701bbc2cd3467fabf75eb78af4cf0a4f8ef7cbc7fd8ef9023d0dd90","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":24.5,"score_display":"24.5","source_stated_rank":32},"run_fingerprint":"8703c3908871055fb24e9205bd4c9b583bddfa542931a882098b53867cc99399","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"50e89c6ce3738d4ba40e146c349aa56839793559a33518104053ea8e09221ec6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.129308,"date_is_proxy":true,"latency_seconds":532.222,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.179},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":null,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"zai/glm-5.2"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"276e9e48ccb07352a0d7380841a202b1aa210b3521796e99c2d98459b02a8023","result":{"confidence_high":71.81084,"confidence_low":67.18916,"sample_count":null,"score":69.5,"score_display":"69.5","source_stated_rank":93},"run_fingerprint":"5d6511064e0346b2c8924ea28ce8c2fea8bf9f12c35e3f5d26ae03a83650184f","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7fa74e2701e012fb5d803f8c","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fad0e167723d8a532e303a4ffcb9c2e2973b4aff83a7904f221a989459c43f8c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.2 (max)","source_effort":null,"source_model_version":"glm-5.2_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"027cbee77755dfbef35c1bc14992098b57107183acda75bedaee14d0c8cac670","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.462962962963,"score_display":"50.5","source_stated_rank":85},"run_fingerprint":"43a7ee3e969c38aefaae5c09055f7d5673a6a329dfc32c059a45f0b3a8d764cf","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6e4feefdd3ae9ca4b4fd201b24742db749af9d68cdd5758494b7769f78efdad5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.2 (Non-reasoning)","source_effort":null,"source_model_version":"glm-5.2_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"91a59625264b4ab23ef1a7708d6b17f99ea0b69f92cfe917e2ac98b703ed56bd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":36.1111111111111,"score_display":"36.1","source_stated_rank":164},"run_fingerprint":"a5ae982c91ba9599e3af4d5c9c2978f963ddec6f39986ecbfd8d9f90baea9d88","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ce870ca70a2599bfcdb0bd0f481f939186585de5529b79aba5dbfc9edd7954eb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (max) · scicode"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-2","upstream_model_label":"GLM-5.2 (max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1a133be34d11397de0627ea4cdcd2c4a212db53e2d268e06ea212a44af965602","result":{"confidence_high":56.877227,"confidence_low":45.407116,"sample_count":288,"score":51.157407,"score_display":"51.2","source_stated_rank":null},"run_fingerprint":"849b1c58ea1d00cb30e8acac76ca7715f51512f539b8e0f4817aa109452e87bf","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d8227d06142f3b9b8ed93af8ff9c16e7d7ff3a2123a4232422248479c9be36bd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.148957","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5.2-max","source_effort":null,"source_model_version":"glm-5.2_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"4c1cf9e791c0b26576c00a51c42d25ce96855e5afbca22f2f22be725da0ad7e0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1010.17,"score_display":"1010.17","source_stated_rank":48},"run_fingerprint":"b63fcef1cb8bbe82639d4d6424fa2d84b3432b8af82e34b0d9dbd13f851c7e11","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"569c354d4e46cb6b936f8f6129fcfe90b4e3044a37d7d50ed9a9a46ba064a0f2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.157666","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5.2-high","source_effort":null,"source_model_version":"glm-5.2_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"84763cd0559ff20aa0bbdf02aaf986e05863eecc60904bf33c205ff1ac6a5b17","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1046.95,"score_display":"1046.95","source_stated_rank":46},"run_fingerprint":"c43dfeeb4e549ba1ec16b396231dbd16885547921236caa92532f0cd6ab2c3e2","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0a11b05baf509af0577d53c4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":110,"at":"2026-06-17","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-06-17","status":"stale"},"measurement_id":"4761ab6a7ae949d1daf7b09ad2ca06eea6f6f72f12f8fab6cfb2d8232d1c0c48","metric_details":{"benchmark_page_updated":"2026-10-02","cost_per_test_usd":8.488514,"date_is_proxy":false,"latency_seconds":3748.167,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/zai_glm-5.2","source_locator":"Results · Overall accuracy","stderr":4.795},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":131072,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"b63af27922d830addbeb8fc995ef235d729fd0923eff5d8f832222d457b2ba87","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.2"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"f4e68fa54f9791056eca1101df37a79920254793b06dabbbe738d29f155dd879","result":{"confidence_high":73.3592,"confidence_low":54.562799999999996,"sample_count":null,"score":63.961,"score_display":"64.0","source_stated_rank":43},"run_fingerprint":"df5c7c376ad3a37bd2612f457f65ae97c5bde5f82fbd3701da70b64155206237","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"89a1f4fcb3124281bdc68332faca83d4ca59174dcf0f12dc1ac2bafb3d88ff7f","metric_details":{"license":"MIT","variance":10.485311761689713},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"glm-5.2-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fe63b21ff94d4726191b76a20d5ecb1e8f89b5fb8580a17d04964354d3982fde","result":{"confidence_high":1611.3476442395197,"confidence_low":1598.65451328539,"sample_count":14729,"score":1605.0010787624549,"score_display":"1605","source_stated_rank":26},"run_fingerprint":"01ae26ead1031d5be14f80a5150fe40fb4e3737ddfea767a53df282a957a9fb5","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b2a4aeb1602f74936ce680c6df20dcb56979151546f4278d711e2606c0264ed3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.2_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"33bccb5157bc2710aa40b97962994f005fc88cd121d41834994eccb70ea7c432","result":{"confidence_high":1607.98,"confidence_low":1578.51,"sample_count":1994,"score":1593.25,"score_display":"1593.25","source_stated_rank":22},"run_fingerprint":"95a66fd94f39b4d6f4c0ba22a98528f55f293bee78d80758a3caf5f09408ca28","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f56480d159626ae61558f237632f0bc380b74eb0f5e5c9ca5f3304a6a9dd239a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.2_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"8c398aaaeb7298f9c247a193edd1cb9b66a708611a8f3497fa2c2efe0634120c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.12,"score_display":"70.1","source_stated_rank":33},"run_fingerprint":"8850d65548427e59eabc2ad13cc00f9ff7c23bdf34802d56587581a1e2c46bbf","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e3bde8e5b0cedb980537d80e5769caf790ba4c99938308041b419c71d9442b4c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.2_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"7aba503d4ee09a5d925728f9aa74f69738fc354ac513f6ca34c8914631567ac8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.31,"score_display":"67.3","source_stated_rank":36},"run_fingerprint":"f40c8faeda7721219dd39c15a603cee9aec2d852ca25e7bf3bb7f2c9710e5150","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_0a11b05baf509af0577d53c4","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":110,"at":"2026-06-17","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-06-17","status":"stale"},"measurement_id":"19d61dbe69c343896ef43245a1c61a168d1de9f283fdde889f2b8489929764c6","metric_details":{"benchmark_page_updated":"2026-09-28","cost_per_test_usd":0.425184,"date_is_proxy":false,"latency_seconds":820.506,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/zai_glm-5.2","source_locator":"Results · Overall accuracy","stderr":0.991},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":48000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"b63af27922d830addbeb8fc995ef235d729fd0923eff5d8f832222d457b2ba87","temperature":1,"top_p":1,"upstream_model_id":"zai/glm-5.2"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"a42980768b9b366b08248a62ef905ea6c849aac8bcb4a1333adeb578383ad2de","result":{"confidence_high":69.73236,"confidence_low":65.84764000000001,"sample_count":null,"score":67.79,"score_display":"67.8","source_stated_rank":32},"run_fingerprint":"dde2503a9a38bd24ec92f2e7e4294987c102a1643b90eddd2aa9077888343f46","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3510da2ea01c0148b737d5353cdbc3cfbe310bc40b33a1961a9ec76ae0fd4430","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (max) · terminalbenchV21"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-2","upstream_model_label":"GLM-5.2 (max)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"b92a15df33d675c4d119b9869131311e27f384e6cbbaa938b8690aee0fb7b0a2","result":{"confidence_high":85.266453,"confidence_low":68.229681,"sample_count":89,"score":77.902622,"score_display":"77.9","source_stated_rank":null},"run_fingerprint":"f7df57c3cde78ba3504ecb7ad642917a034a59952c53b018fcdd2ad2aa6ede99","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (max) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0f99a38b109e247d2818f498cff6f61c1271551e0d3ec01dfe0e309d14259c8f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · terminalbenchV21"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-2-non-reasoning","upstream_model_label":"GLM-5.2 (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"b6c4f79436ceaa94f135cedd9a9a83f6c6ec44f8d1df841bcff99a39960bf778","result":{"confidence_high":61.780906,"confidence_low":41.450403,"sample_count":89,"score":51.685393,"score_display":"51.7","source_stated_rank":null},"run_fingerprint":"0816415277d6ad26488e3660143cd8e569e822f2fe95c6f242c962258ff9e98f","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-3","metric":"accuracy","name":"Terminal-Bench 3.0","release_id":"release_ceb36f25882d8cea2c9d084a","split_or_window":"official-74-task-leaderboard","unit":"percent","version":"3.0 · b936386b-4afd-4ad7-aa7f-6b52bcab9853"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":53,"at":"2026-08-12T21:39:34.661007+00:00","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T00:52:13Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-12T21:39:34.661007+00:00","status":"fresh"},"measurement_id":"f25aef2b4ae365ee4e57dc7b13e846600b54cab946f4165a35d2b864a2b8e986","metric_details":{"accuracy_stderr":0.97,"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 74 scored items with a 95 % Wilson interval.","leaderboard_updated_at":"2026-09-03T00:07:57.08561+00:00","row_updated_at":"2026-09-03T00:06:55.46515+00:00","total_cost_usd":3401.76,"total_tokens":3276045131},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_organization":{"label":"Anthropic","url":"https://www.anthropic.com"},"agent_url":"https://claude.com/claude-code","dataset_version_ids":["b936386b-4afd-4ad7-aa7f-6b52bcab9853"],"leaderboard_row_id":"b3c7db0e-0289-4fa5-ac3a-da600b15a72a","model_release_date":"2026-06-13","model_url":"https://z.ai/blog/glm-5.2","n_trials":370,"source_row_date":"2026-07-19"},"run_count":5,"scaffold":"Claude Code","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"8193d21ad0eb789a1ec26ea50fbd3379bed4aad4e8b492bd1b886fd1b07a3897","result":{"confidence_high":11.991954,"confidence_low":1.67015,"sample_count":74,"score":4.59,"score_display":"4.59","source_stated_rank":12},"run_fingerprint":"6d26e312adf3154eae3de6b419df7d390cc6f7fa31e2c86f95cfac4740fa7613","source":{"content_hash":"d56d142efe23d510fb6c7a3723540298c3c5f6810acb6ff6b87070e5e7022e9a","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-03T09:20:24Z","homepage_url":"https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?tab=leaderboard&leaderboard=3-0-0","id":"terminal-bench-3","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0 benchmark; public Harbor leaderboard metadata","locator":null,"name":"Terminal-Bench 3.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://ofhuhcpkvzjlejydnvyd.supabase.co/functions/v1/leaderboard-read"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":73,"at":"2026-07-24","basis":"evaluation_at","evaluated_at":"2026-07-24","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9d344d177511c750cdadd1b57abe1af0890faa3bda97bfd09c2e9c9f5fa4accb","metric_details":{"cost":null,"pass_2":25.257731958762886,"pass_3":18.298969072164947,"pass_4":13.402061855670103},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"glm-5-2_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"8ae31a68885f85099acfd11744c8b6482786a6c228b5cf38146830c9fac67e2e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.11340206185567,"score_display":"37.1","source_stated_rank":null},"run_fingerprint":"db11e6754d914c727876392db98e0567d1aa3ebe39202438cec4c222ca6f2c8e","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_731f1098dce287a031491f2b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"12363b336cc7b3d52b822b412fc666a7df758dfa6926df30a5338fd435851928","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (max) · tauBanking"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"glm-5-2","upstream_model_label":"GLM-5.2 (max)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"4c59c099162c4a01f2649c2a1a8acc544f302d9a3204752aadb34fb996add352","result":{"confidence_high":44.52984,"confidence_low":25.918863,"sample_count":97,"score":34.639175,"score_display":"34.6","source_stated_rank":null},"run_fingerprint":"945012d71431e412d45ec6b0a12aa17eb6e71c3e02b18d5f4834f1ef403b6a3e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (max) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"831fa1a4991c89f207a6975ff9a3f940cce4b2a91a648e61cbeb2262dbfbf12b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-06-16","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · tauBanking"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"glm-5-2-non-reasoning","upstream_model_label":"GLM-5.2 (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"3c4a864d4f0f5d23034868582bddb551d31830a85194d935521af024d1e7daab","result":{"confidence_high":25.359201,"confidence_low":10.579935,"sample_count":97,"score":16.701031,"score_display":"16.7","source_stated_rank":null},"run_fingerprint":"41860f3a53cfebd680362272a96468d1a90a0ed26f1631cddb73cb8fa3bff94f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":76,"at":"2026-07-20T17:09:57Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-20T17:09:57Z","status":"fresh"},"measurement_id":"641eec0bca9d9dc4f0cf0c32c5bb91f9af1411e2c078130c05ab17d6b5201b12","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.6,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=glm-5p2; createdAt=2026-07-20T17:09:57Z","source_row_created_at":"2026-07-20T17:09:57Z","task_pass_coverage_threshold_percent":75},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"glm-5p2","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"36e00d06eaa902ac2df7d0c8d943d498a5d6b6fa8e47d2c808ff12d014484e7e","result":{"confidence_high":80.39999999999999,"confidence_low":75.2,"sample_count":1000,"score":77.8,"score_display":"77.8","source_stated_rank":3},"run_fingerprint":"5e0864ab6ef7f2afb91041bc7331f4de998ef65a5781414dadbb7b630f7c6a58","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=glm-5p2; createdAt=2026-07-20T17:09:57Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_485b19c39bb21348a203eb53","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"532e75e8fd52eb4b759e3fc92680dcddc82921b8f64098354d335a1ea326a772","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.522","mean_standard_error":"3.5","model_release_date":"2026-06-16","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.6,"upstream_source_url":null},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.2","source_effort":null,"source_model_version":"glm-5.2_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"f96ad31f7f6ddc65568919ade40155072350011ba1ded5cf869cfb9a78b9eb50","result":{"confidence_high":42.656,"confidence_low":28.544,"sample_count":480,"score":35.6,"score_display":"35.6","source_stated_rank":17},"run_fingerprint":"3873a5b8efed42f9c8715b6b8ae47a6a2e154d99e2c4d2e6d3a6ad86bd8cdf5a","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_27cd4b5c9a4a27c645ccfbf4","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8c61de07cf79477f6a81e58f9cc083e97e5d92f34c2e98e10f52c2bf33dfbff5","metric_details":{"confidence_level":0.95,"license":"MIT","session_count":86172},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GLM 5.2 (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fce8c13196c1026ced56ef0ffca4db976510b8c1a37a2f1f481b1c9f9ffa5e30","result":{"confidence_high":0.04197633165184164,"confidence_low":0.027614565013024867,"sample_count":6684620,"score":0.034795448332433254,"score_display":"0.0348","source_stated_rank":20},"run_fingerprint":"e85dbdccbe0ba2d1d8910531de4a36acf03b9b30ea876f86116000d72ac74c74","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0e270924d1e36211583b0d8bc5af21c56ee5f62640e1bc36d996cd5f6ddf3c5f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-16","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.2","source_effort":null,"source_model_version":"glm-5.2_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"30bb57dda941478e098ec5e225aa9517aeeff59a03ace8296132a1a40706b998","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":8313.778333333335,"score_display":"8313.78","source_stated_rank":8},"run_fingerprint":"2e054fa12a991a3a6937f9a0f0d43bc21d27193f8d43a6e41a41b9529fc08bbf","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20ab07deb4f86981c9497a7c","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b2a73493b3d9c7582074c7060ae82b1ead45f91734c2dc96ad724511f55971e5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-16","reported_confidence_interval":"-21 / +21","source_locator":"Leaderboard models table · GLM-5.2 (Max) · gdpval"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"glm-5-2","upstream_model_label":"GLM-5.2 (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"35bf4e7940db691ad41d0946b628a8a1417c3d185783bf6f3d7267b7144f6ca4","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1357.52,"score_display":"1358","source_stated_rank":null},"run_fingerprint":"a34b3072700988d81eb40b1c95e74c0d93ecd394c06f00fc73f44a2cc8238e79","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a6e3ca6e600d7d7ad3656281","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0ab45abbce8c772e7ab578456642a2ed53cd60f9d0e060494f2bea7b95c783da","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-06-16","reported_confidence_interval":"-27 / +27","source_locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · gdpval"},"model":{"id":"zai/glm-5-2","name":"GLM 5.2","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"glm-5-2-non-reasoning","upstream_model_label":"GLM-5.2 (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"0983cc9f30d00a3c41357acf8c62b20155c45beff98896731514eed70d82bf71","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1233.0,"score_display":"1233","source_stated_rank":null},"run_fingerprint":"b4207c0c0c4f59b844bd2b7ee6da5431f337dee4bdd735eb3d9311a747cce872","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.2 (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2dc4d509d97ba2d565e36ba8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b1c89bfea230d90eaeb3ca336608eec00242cd9dd0ed27e0b3d6f73ae3159c78","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-20"},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Kimi K2.6; model release: 2026-04-20","source_accessibility":"Open weights (unrestricted)","source_model_name":"Kimi K2.6","source_model_release_date":"2026-04-20","source_model_versions":["kimi-k2.6","kimi-k2.6_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"644ca544e135db7d2e196bf02145906ed178d5f81c1fef1d19fbef842b4d18e2","result":{"confidence_high":152.81,"confidence_low":149.12,"sample_count":null,"score":151.05,"score_display":"151.05","source_stated_rank":46},"run_fingerprint":"15ed2108184e79c575cbc133b66906f8274d9675a9d2415fcd2c2fe9481b286f","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b310499954e2e03883d1bab8","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e55368d2735c53e08b84afefaa9c04887feee463032bc96486a9f63410bc2488","metric_details":{"license":"Modified MIT","variance":5.111051285621139},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1465.390454630528,"confidence_low":1456.5284248304947,"sample_count":39936,"score":1460.9594397305113,"score_display":"1461","source_stated_rank":58},"run_fingerprint":"398d977d9910aabf849f75704e2ceb41bd66026a47ee61851266b6f5baebb462","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_efc5f41143198675a7b4f23e","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a6b42c46bb9a349f3b7bb04c04d7c65c29feb90db88a0bf56fed78e645ab1b22","metric_details":{"category_scores":{"Agentic Coding":46.919333333333334,"Coding":78.56700000000001,"Data Analysis":65.13433333333334,"IF":64.35825,"Language":75.14133333333334,"Mathematics":84.27625,"Reasoning":79.375}},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.53878571428572,"score_display":"70.54","source_stated_rank":54},"run_fingerprint":"baa2f49b8882b3904368edd0845a2b88c4c6000680293410fa1c33b9fec8eb7f","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_42ae63213d558eb4662e56b8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":168,"at":"2026-04-20","basis":"evaluation_at","evaluated_at":"2026-04-20","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e4bcdb2641b42e6fbc95ff19da97b21a0b778ef140dfd99b395e50ea1c2efe66","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/moonshot-kimi-k2-6","source_model":"Kimi K2.6"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fba5f6b10977c74d12a0a51f51615d7e5547d98b2fd12ae73799ca2e93a9ceeb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.9,"score_display":"29.9","source_stated_rank":26},"run_fingerprint":"ec8de890b40885f2460eb1bd33be7a7283c41031722853f16d9a9c4f9a491dc5","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_68921aa42095f28a6654781d","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":168,"at":"2026-04-20","basis":"evaluation_at","evaluated_at":"2026-04-20","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c870d961c96d863973d23e952ba59ac1095e4e8da45231efd95385cf95dab89b","metric_details":{"modality_lane":"not reported","notes":null,"num_parameters":1026879376368,"pull_request":null,"result_file_url":"https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/.eval_results/hle.yaml","source":{"author":{"_id":"63e0eea7af523c37e5a77966","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1678663263366-63e0eea7af523c37e5a77966.jpeg","followerCount":485,"fullname":"Nathan Habib","isHf":true,"isHfAdmin":false,"isMod":false,"isPro":true,"isUserFollowing":false,"name":"SaylorTwift","type":"user"},"isExternal":false,"name":"Model Card","url":"https://huggingface.co/moonshotai/Kimi-K2.6"}},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.7,"score_display":"34.7","source_stated_rank":34},"run_fingerprint":"6b3937000625cc12e0c1142bd77e0927df78529f87acd3282b5482be71d0147c","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1d07d7296f584234d504fa94","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":168,"at":"2026-04-20","basis":"evaluation_at","evaluated_at":"2026-04-20","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ec1a018e726812001d015857522a2b2657c0566d29b3bb52849f39244429a0d7","metric_details":{"modality_lane":"not reported","notes":"With tools","num_parameters":1026879376368,"pull_request":null,"result_file_url":"https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/.eval_results/hle_with_tools.yaml","source":{"author":{"_id":"63e0eea7af523c37e5a77966","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1678663263366-63e0eea7af523c37e5a77966.jpeg","followerCount":485,"fullname":"Nathan Habib","isHf":true,"isHfAdmin":false,"isMod":false,"isPro":true,"isUserFollowing":false,"name":"SaylorTwift","type":"user"},"isExternal":false,"name":"Model Card","url":"https://huggingface.co/moonshotai/Kimi-K2.6"}},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.0,"score_display":"54","source_stated_rank":8},"run_fingerprint":"357c3b8672a4ce9cc85b9b68fdbc1771958c227e7839d4da2d25fdcb40c02636","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1d07d7296f584234d504fa94","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T21:35:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T21:35:30.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9383874071cc2713185f2fd1f7e72b68eaa42a13b7d033f9fb2d749744d439f8","metric_details":{"best_score_across_scorers":"0.349","model_release_date":"2026-04-20","stderr":1.5080663991563052},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hHnQVatVTbQjM4sfrjw7GH","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"09143fdf56d84fdffc3344df3475d5a69d20be0048770624622b4be3711fda75","result":{"confidence_high":37.85581014234636,"confidence_low":31.94418985765364,"sample_count":1000,"score":34.9,"score_display":"34.9","source_stated_rank":51},"run_fingerprint":"bf1324e22e0a0e300921d65898a7bfc9281ad066a4692208d45e2a4a10383e70","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_97380cea8f577d9e27da9479","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bc908ac20ead136fe11c0c2a2d9ebbbf4ac18904a4ca6f91a2230ec2ab0f9893","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.080129,"date_is_proxy":true,"latency_seconds":544.744,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.997},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":89.899,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.6","zero_shot_accuracy":88.384},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"bd1d1f5b31a8116d52d703150912df0a60fc3134bc0eb5d6aab8f6732d16995c","result":{"confidence_high":93.05611999999999,"confidence_low":85.22788,"sample_count":396,"score":89.142,"score_display":"89.14","source_stated_rank":31},"run_fingerprint":"986a799d6d3cfaa65bdd3cb542c75ed5166e7aaf4f9cd4bdbc0b96438ee112e7","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_34a077e25a13de6727203c0f","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":156,"at":"2026-05-01T17:25:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-05-01T17:25:36.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"9333a267a59805459e9d293e0f980e044628bcdd62a6a5461684b6562209d2cb","metric_details":{"best_score_across_scorers":"0.9078282828282829","model_release_date":"2026-04-20","stderr":1.7174516286666601},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LjgJoDosTLkiGVT59smA7q","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLjgJoDosTLkiGVT59smA7q.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/LjgJoDosTLkiGVT59smA7q.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"c5d12474fe2b27cd14627b418f9c67500f78e833a6c23a7556e8ec0db78ac640","result":{"confidence_high":94.14903347501495,"confidence_low":87.41662309064164,"sample_count":null,"score":90.78282828282829,"score_display":"90.8","source_stated_rank":38},"run_fingerprint":"ca7837fa411857e7a0f65f17aefb4f5c1781ba645ed0ef11875f2259bb3d7194","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8961f724a1b27a4e755fad98","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef6db9d4d3467213d1d6f77619f60388d138ae8d983e7241ebe3fbf1b557452e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.018214,"date_is_proxy":true,"latency_seconds":156.347,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.33},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.6"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0109db64ab383bcde8ff886b1cde9e0f9d5d9a7c0a9c49d53cd049ee7f9dbe53","result":{"confidence_high":88.2188,"confidence_low":86.9252,"sample_count":null,"score":87.572,"score_display":"87.6","source_stated_rank":25},"run_fingerprint":"b7d5baa16d7c18fe05a61a749f0b1ebde02b354ec01d9f1b71f342bf59d2f317","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4b0cec355eeaeb214599a32e","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":156,"at":"2026-05-02T09:08:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-05-02T09:08:49.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"35b02867f6f345e0a3913ac24ca6101d2909977ebe6f704defd61113bd1c1d7e","metric_details":{"best_score_across_scorers":"0.9611111111111111","model_release_date":"2026-04-20","stderr":2.3777636174410794},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BTeRdhgt6TFBj6QkGX2j6P","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBTeRdhgt6TFBj6QkGX2j6P.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/BTeRdhgt6TFBj6QkGX2j6P.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"eee7ad52081a22a866018fdb36a96a8d45cf0b2c81142142245a800fbef0f45f","result":{"confidence_high":100.0,"confidence_low":91.4506944209266,"sample_count":45,"score":96.11111111111111,"score_display":"96.1","source_stated_rank":34},"run_fingerprint":"3a0c9068e73507dac3d1d379e36ea7f73aa58bc48c93d17a0eb0d3faa4ca06cf","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_13dc2af2a60f2cc07172f0f7","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T19:47:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T19:47:36.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e96c21968a0d541812913a74b6d26a4735a9e6de75484c2eee41cb0e98d89249","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5719298245614035","leaderboard_private_problem_count":285,"model_release_date":"2026-04-20","public_example_count":10,"stderr":2.9360921879029935},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"BxrNfyMffkVUq4n7U5eK3Q","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBxrNfyMffkVUq4n7U5eK3Q.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BxrNfyMffkVUq4n7U5eK3Q.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Kimi K2.6","thinking":null,"upstream_model_id":"kimi-k2.6","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"fd03000a890c14001f3d5c700ecc96f6eb91519f1ac9df62125290f68ae9c967","result":{"confidence_high":62.94772314443021,"confidence_low":51.438241767850485,"sample_count":285,"score":57.19298245614035,"score_display":"57.2","source_stated_rank":39},"run_fingerprint":"6c07ef00964d085e64161b1fc0560623cf18b142283f64cff51262346f5475b3","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d1fab5b0d9cdb6adb08d2014","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T19:47:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T19:47:36.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"cac640ce6b9b28e66049bd5dff516f1dfbbfc37f8812055ccf62bc5bdf7edd3f","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.2564102564102564","leaderboard_private_problem_count":41,"model_release_date":"2026-04-20","public_example_count":2,"stderr":7.083413480167724},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"eVKA2eavdRAbSN6AnzyBxA","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeVKA2eavdRAbSN6AnzyBxA.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/eVKA2eavdRAbSN6AnzyBxA.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Kimi K2.6","thinking":null,"upstream_model_id":"kimi-k2.6","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"79753c91c17a8c32905f7dda5872e1340706d508f05b9928e1eb76a02c7431dd","result":{"confidence_high":39.524516062154376,"confidence_low":11.7575352198969,"sample_count":41,"score":25.64102564102564,"score_display":"25.6","source_stated_rank":41},"run_fingerprint":"610dde98a40310167437df9a21aae70950da03032b3ea1bd9ab3d9bc15fa43cd","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20fdd50658cc979ced3365dc","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3c50c190ffd7009cc0e482f118d02ac0253225beb82de59afa5fe70a4c480ca2","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.034867,"date_is_proxy":true,"latency_seconds":241.908,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.827},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.6"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f5b010081531b16ac7cfcf2165a5b4483ca5a16cc3d31b0b9d3dde0f9f7dd838","result":{"confidence_high":87.92192,"confidence_low":84.68008,"sample_count":null,"score":86.301,"score_display":"86.3","source_stated_rank":21},"run_fingerprint":"a154c091b65df7cc0c5b3b82992406e8af5b65504bf9e921eec79ddc9820b75e","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f9ebb9b70f07c7fd44aa7c3f","provider_config":{"source_id":"vals-mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_1dbdadbee92bb7f366d1025b","split_or_window":"standard","unit":"percent","version":"MMMU-Pro official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":168,"at":"2026-04-20","basis":"evaluation_at","evaluated_at":"2026-04-20","first_observed_at":"2026-08-29T23:52:50Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ef61ce01fa6a4cf5753b10f157ecf027ebfd2c0ba84d8acd862a5775aa1c56be","metric_details":{"num_parameters":1026879376368,"pull_request":null,"result_file_url":"https://huggingface.co/moonshotai/Kimi-K2.6/resolve/main/.eval_results/mmmu_pro.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/moonshotai/Kimi-K2.6"}},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9225c08e521da63df44b0ad31230f50489ffbee930c8fb4aaf2f0f8dbf6fce3f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.4,"score_display":"79.4","source_stated_rank":2},"run_fingerprint":"ea323c78301e08a929f943cb2d8ad89507228fb9364162d8e957684f26f5812d","source":{"content_hash":"bc590a866070c9674cba0de4af7d92af6e0921bd2db419fca54624e00d73400b","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-10-05T12:33:31Z","homepage_url":"https://mmmu-benchmark.github.io/","id":"mmmu-pro","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"MMMU-Pro","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/api/datasets/MMMU/MMMU_Pro/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_61b05e972b84845635f371a5","provider_config":{"source_id":"mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5d66bd25806023ea3dca36ef5eae2cf80e5b929c4f8631396fe3d7fd04baffbc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.6","source_effort":null,"source_model_version":"kimi-k2.6","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"1cfcaea211044e0eecdcb06ccec090730c95c6d8d8d3b891766987ecdb5a5c22","result":{"confidence_high":16.669402,"confidence_low":3.642296,"sample_count":71,"score":8.0,"score_display":"8.0","source_stated_rank":101},"run_fingerprint":"6f23cc31f5ecc640fc683af2d190c967f884b00fad2c40a7375afa2296accdad","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_272add6740d961e9d97d445b","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":150,"at":"2026-05-07T17:44:35.000Z","basis":"evaluation_started_at","evaluated_at":"2026-05-07T17:44:35.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"2fef6ccec4d30b71815a4d07b8416c11511589e3f5eafdb59a13a951524f6124","metric_details":{"best_score_across_scorers":"0.26","model_release_date":"2026-04-20","stderr":4.4084400227680804},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4MWuvJB4YXGnGz27Y74cqW","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4MWuvJB4YXGnGz27Y74cqW.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4MWuvJB4YXGnGz27Y74cqW.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a95eb1c98489b15b981c73260c5673b62f824f56aab6dc9b3fd70ab6a6732571","result":{"confidence_high":34.64054244462544,"confidence_low":17.35945755537456,"sample_count":100,"score":26.0,"score_display":"26.0","source_stated_rank":62},"run_fingerprint":"d12d2833a49d6bdb1a64f2c844ac24423514daa7cd52c70d2279186b75945d9c","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5933a601e7c8c1587b769e58","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fd8c33aac4e98299d5de71918e5361e0cb3d9a8d50d311f15777fb14937f1401","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.27474,"date_is_proxy":true,"latency_seconds":519.829,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.906},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.6"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"4f90574b798153d95aa56ea8cf26457e4ccf638e0b114ef2397d9c56de0763b9","result":{"confidence_high":79.93576,"confidence_low":72.46424,"sample_count":500,"score":76.2,"score_display":"76.2","source_stated_rank":41},"run_fingerprint":"4baf7d3d026545c3af842a4341d239f94d584e0712332b7b492bb27f10203ede","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_55a0f6cd9901fb49f5645d5d","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":150,"at":"2026-05-08T10:21:24.110Z","basis":"evaluation_started_at","evaluated_at":"2026-05-08T10:21:24.110Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c3667ceb2cb8a4ee4c8d72292a84344dae269bdbab8fd61c83875028b6c2a9bb","metric_details":{"best_score_across_scorers":"0.7665289256198347","model_release_date":"2026-04-20","stderr":1.924895909142592},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bUPtAK7spEwvmiC8pi6afc","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbUPtAK7spEwvmiC8pi6afc.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bUPtAK7spEwvmiC8pi6afc.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"490d5838b7d48e6778d7f3d931f7843aebb721d8c0599841055aa95915305542","result":{"confidence_high":80.42568854390295,"confidence_low":72.88009658006398,"sample_count":484,"score":76.65289256198346,"score_display":"76.7","source_stated_rank":9},"run_fingerprint":"5cced63a43b602e25252b26026b93f6b0c3f80b9038239dadacbfe071ef879a0","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d0c4d4018f2c9fc2e624063d","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"b21ad5872a6e89f68f9d0b98ad4b8b4d9720f2264d892a74b71629c9ab867ec3","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-03-01","pass_at_5":64.54545454545455,"potential_contamination":false,"sem":1.2662171161076474},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"Kimi K2.6__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"4d1be04b07e6b95bf0f76baf433081fb6639c9fb4e627252e3d479715ccbfbaa","result":{"confidence_high":49.02724009302553,"confidence_low":44.06366899788356,"sample_count":null,"score":46.54545454545455,"score_display":"46.55","source_stated_rank":null},"run_fingerprint":"be694bdefd40775a51817ae697c4f62023561dcb56ef9ada9173e1300a3291b2","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85d245e50cf88f18729906fb","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_523506ddd6112c870b565682","split_or_window":"moonshot-sweagent-derived-10-run","unit":"percent","version":"SWE-bench Pro · Moonshot in-house vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":168,"at":"2026-04-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:18Z","last_confirmed_at":"2026-10-05T12:33:36Z","observed_at":"2026-10-05T12:33:36Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-20","status":"stale"},"measurement_id":"45a41f77ae0f9d288b7aa3c190d6d0dceb16131ce8607c64cc048e998bb036bd","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · Evaluation Results · Coding · SWE-Bench Pro row · Kimi K2.6 column; footnotes 1 and 4"},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Moonshot AI","evidence_verified":true,"freshness_proxy":true,"protocol":"thinking mode enabled; in-house SWE-agent-derived framework; temperature 1.0; top_p 1.0; 262,144-token context; mean of ten independent runs; exact task count and revision not disclosed","reasoning_effort":"thinking","result_published_at":"2026-04-20","source_content_hash_method":"raw_bytes","source_content_sha256":"ef96b98f7b43995f09151578e574ed0a02b177fca5e793c2cb9940b6cb9e6617","source_published_at":"2026-04-20","tools":"bash, createfile, insert, view, strreplace and submit"},"run_count":10,"scaffold":"Moonshot in-house framework adapted from SWE-agent","tools_allowed":"bash, createfile, insert, view, strreplace and submit"},"protocol_fingerprint":"6ab6b6a3dc150f97c7a80ab9f0845df9884468896fe764d3d381f4d8e0d48d1f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.6,"score_display":"58.6","source_stated_rank":null},"run_fingerprint":"e1f5e5024e9326a02c21c595d2d51b25c475c5576743a9dbd744ea7d6df10ba2","source":{"content_hash":"891cb6d4c1c396794bed81573b3dd5f9e0325e1003ae957ac82a71e2189f7b2f","fetched_at":"2026-10-05T12:33:36Z","first_fetched_at":"2026-09-03T09:21:20Z","homepage_url":"https://huggingface.co/moonshotai/Kimi-K2.6","id":"moonshot-kimi-k2-6-card","last_fetched_at":"2026-10-05T12:33:36Z","license":"Modified MIT model card; aggregate factual scores with attribution","locator":"Immutable README · Evaluation Results · Coding · SWE-Bench Pro row · Kimi K2.6 column; footnotes 1 and 4","name":"Moonshot AI · Kimi K2.6 revision-pinned Model Card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/moonshotai/Kimi-K2.6"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_42ae63213d558eb4662e56b8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"4c59aa931f24fa2b2b3329887e14a8cac15d6416876bf8f54697ea33b6e12f63","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.27","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-04-20","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"56","tokens_per_task":"24783","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi 2.6","source_effort":"","source_model_version":"kimi-k2.6","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"a08fe910aff6b40aaf8a4c58d278a47367e962192ec1e47bc7a5f3ed0587ecc1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.6,"score_display":"47.6","source_stated_rank":68},"run_fingerprint":"0699c0d0a92d603756da43061c1f60d4f6bc3bcf8af6b4780dc176ec2cd0e42f","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_53c4811e8695a8656faeac6a","provider_config":{"source_id":"epoch-cursorbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7057bfd699b11280dae15d850a7a688d0335312f9784f5aad916922d0a6ea95c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.06543,"date_is_proxy":true,"latency_seconds":529.063,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.972},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.6"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"63ffcde8baf25909147169367e67d9dcb44bfbd13fda938239dac5b42b9c5924","result":{"confidence_high":88.67612,"confidence_low":84.86588,"sample_count":null,"score":86.771,"score_display":"86.8","source_stated_rank":20},"run_fingerprint":"7d4a5389328d5e3d1a5f101933beff7c6eb40308d3301ade257087624904ef56","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0921b0a660bb2fa8eba5dd75","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"02a83fc99b596b30b561fed01f513e89632b78356abb4496569adf03a70c8cd0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.6","source_effort":null,"source_model_version":"kimi-k2.6","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"c96e38cd745c305f2097b7074864adf7246c8a809c41b1eedb41bdabcef11817","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.4722222222222,"score_display":"53.5","source_stated_rank":62},"run_fingerprint":"9725b3d20d5fb8948aa44dd58e5573203186dd41518dc55a0005f73dc0a8760f","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_96286f612a6bc490a7c6da8c","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"727db0ee558e2b0d7725eeb148c679ce516c57360e3d972654674fb28f3285bf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.196861","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"kimi-k2.6","source_effort":null,"source_model_version":"kimi-k2.6","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"3703c854922fd87933ffac6a03927594645fb65bf266479163effe7a72cbc1cf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1092.67,"score_display":"1092.67","source_stated_rank":43},"run_fingerprint":"ad4dd09733143c55acf1bf82becbc6d4259b9653b8708a7c22c8f21daf243bb8","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b51293b6b43d208ca931a702","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fd582436e9527d948462d91a9997b395c04820fdaf95e80a660d93819e9b5008","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.925351,"date_is_proxy":true,"latency_seconds":2967.768,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.912},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.6"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"4547e71d6f5f6b19438ecd44025c5bca30d8fefb1ddcc9119a1dfe26380f0e9d","result":{"confidence_high":47.518519999999995,"confidence_low":28.263479999999998,"sample_count":null,"score":37.891,"score_display":"37.9","source_stated_rank":62},"run_fingerprint":"c75a538ca1feaa977cc4bc33f2c15541b2f81da1b3f739ccebcf48bc739ed5f5","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9eeb07a0de769530f935f98c","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9d41b4a7c03dc69c224a47454608d9e43ac896105f9bb601bcbbe4a51acc0f7c","metric_details":{"license":"Modified MIT","variance":13.550694080600643},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"kimi-k2.6","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6570a0c7994e646d82fcf0a41c316c3010f9b7e09b7f4b85f9f2114455a98281","result":{"confidence_high":1515.9555915338444,"confidence_low":1501.5258398655099,"sample_count":10208,"score":1508.7407156996771,"score_display":"1509","source_stated_rank":51},"run_fingerprint":"9da2011fdf49f605ff6943169c9d7e283d888769148fa43f674ed068227cb43e","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_768253bf922f44c1efe3bf53","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d6d9dc9475af95344a55b87eb3ad3ac010110933111dfa90fe8514fb7c03e117","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k2.6","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"e8aa5e274e94e1105acddedda651a0436a30c31f440b3024845ac00539bcbcb5","result":{"confidence_high":1516.18,"confidence_low":1501.37,"sample_count":9585,"score":1508.77,"score_display":"1508.77","source_stated_rank":49},"run_fingerprint":"fba37a56414ec6c1b1742eb78f125df3885bc1254e207b8827abf2de0f211f84","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_004b49a21d19fd73b3aa0513","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"115d7471851a478bd55178a5929ac78bf514d4a5d9439d589c4f50ade32c37ed","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k2.6","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"7c243afe23aa63e3900363e414753a05d10990b90cc42da92099149355363aea","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.86,"score_display":"55.9","source_stated_rank":59},"run_fingerprint":"ac8eaad01259b173c6901c7a042dd96e24d262096f26859cd702b62b61d32eab","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3011d20cf3046664a1d12e36","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"28ceff3268abddf92e8525bcded17512030209acc828798eb4beff674a2f3be2","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.138449,"date_is_proxy":true,"latency_seconds":932.358,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.085},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":256000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.6"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"19bc6d91c06579a87a78863e1b2043de3ed273839b879f9dcb75932304d85a2b","result":{"confidence_high":57.6446,"confidence_low":49.4714,"sample_count":null,"score":53.558,"score_display":"53.6","source_stated_rank":54},"run_fingerprint":"baf104806d2fd7ddebfcdf02c3c34e2f779298911b752382806fe72fc6b2f337","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_807bb82ee74af1856495dc68","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"e5407904f4372dbe8c907a5b04afcd16ae258be8fe44092098eec21c49a3c844","metric_details":{"expected_trials_per_mode":261,"normalized_gain":31,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":20.6},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":62.8,"confidence_low":45.2,"sample_count":87,"score":54.0,"score_display":"54.0","source_stated_rank":8},"run_fingerprint":"398a6046fe0df64a1b6805979de434544381eb78c043bc13dfe324816a282ce6","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c8716685b104fcbca109c687","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"2eb6833ba12cb86bbb26986e97cab81c07991ac5253e8f6dbc1f4ee99a9d32f5","metric_details":{"expected_trials_per_mode":261,"normalized_gain":31,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":20.6},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":41.199999999999996,"confidence_low":25.599999999999998,"sample_count":87,"score":33.4,"score_display":"33.4","source_stated_rank":11},"run_fingerprint":"adb297734b71ed49d33a62f92c63a780e6f48f157f925b90acf73a43ddc6c382","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c8716685b104fcbca109c687","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"0864242a357764a62650531d143772b80affad9bf21dd6fe2eae83f9e621614c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.332","mean_standard_error":"3.0","model_release_date":"2026-04-20","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.8,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.6 (Thinking)","source_effort":null,"source_model_version":"kimi-k2.6","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"8a0f72b05e17fd169b5a9e36a0024dcb40b40df8eb50a6975ee02984eafa1e28","result":{"confidence_high":24.387999999999998,"confidence_low":13.411999999999999,"sample_count":480,"score":18.9,"score_display":"18.9","source_stated_rank":38},"run_fingerprint":"f57108b1dc67c3a1de3e0a0c74c28d78ba361705d7516a1d05d84508151a44dc","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_42ae63213d558eb4662e56b8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8301161f961a2b7786634b029121bb74af0158f80d3513590668db0cfb1d2efa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.6","source_effort":null,"source_model_version":"kimi-k2.6","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"7a300a06689c1bfa973d94d09dc598138039797d33b8ab97c0f89de019eee6bf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":6204.570000000001,"score_display":"6204.57","source_stated_rank":16},"run_fingerprint":"e9151fc83859a827c1f68d5c6020ba1fc96ba87d1f5055f38cfddd4d4f5632b8","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8c49ed901c348b23f62acd44","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"81aba01c81e8ed1b3fc9fa682f45fd1392ace66be2b3d8a17dc00126ea196b86","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-20","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Kimi K2.6 (Reasoning) · gdpval"},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"kimi-k2-6","upstream_model_label":"Kimi K2.6 (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"4c3b090c7ca0b7ccf6045418d23e8fac845b8890792e7065dbf8103cb3c454fd","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1026.09,"score_display":"1026","source_stated_rank":null},"run_fingerprint":"6f1fee2486142373c7ec7975a59706c38c95f079380fbb3573db215f6835c584","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.6 (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_42ae63213d558eb4662e56b8","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"49a936fbcac74fdf01c3bb5184ed3e45f10d1e163ead8c4c29b1feebfd25b170","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"moonshot/kimi-k2-6","name":"Kimi K2.6","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi 2.6 (enabled)","source_effort":"enabled","source_model_version":"kimi-k2.6","source_row_date":null,"source_scaffold":"standard","upstream_leaderboard_url":null},"run_count":null,"scaffold":"standard","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"0d8f505dbc71ac537e0aa87c689661ebe93d7810bf90905213a1f0cdaccf5283","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4.6,"score_display":"4.6","source_stated_rank":14},"run_fingerprint":"7123481ff7f96afdff39ed6bb2021c0282273572aefb9f10c1a052a4ee562ec0","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7a58f8d92e38556c6d31424b","provider_config":{"source_id":"epoch-osworld-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b1b75ffeb600ec409e5e1b9fae756b39289ed08e2194b72ea8abe223ec8edadd","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-10-07"},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5 Pro; model release: 2025-10-07","source_accessibility":"API access","source_model_name":"GPT-5 Pro","source_model_release_date":"2025-10-07","source_model_versions":["gpt-5-pro-2025-10-06_high","gpt-5-pro-2025-10-06_unknown"],"source_reasoning_efforts":["high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cef4530569a6ad47fd54e38f836601b325d4d3feae4512af47c7e654a6e0413b","result":{"confidence_high":152.8,"confidence_low":148.45,"sample_count":null,"score":150.26,"score_display":"150.26","source_stated_rank":47},"run_fingerprint":"c43c9b57fd466f21c145be9b61fdef53bd5e4c2e3851f36978ea7f75e332885b","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e62c7adc554af7c375c9ce06","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":332,"at":"2025-11-06T22:48:20Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-06T22:48:20Z","status":"stale"},"measurement_id":"3528c4ec3716b1d54d33e762b7906df2c308feb10e54c1f9f721dd71329a3f22","metric_details":{"calibration_error":49.0,"confidence_interval_half_width":1.82,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=gpt-5-pro-2025-10-06; createdAt=2025-11-06T22:48:20Z","source_row_created_at":"2025-11-06T22:48:20Z"},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"gpt-5-pro-2025-10-06","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"239eb4f55fb8092bc928ed72964797e4e3a99ec7f944a5d778c635482e4abd16","result":{"confidence_high":33.46,"confidence_low":29.82,"sample_count":2500,"score":31.64,"score_display":"31.64","source_stated_rank":9},"run_fingerprint":"d8382df2f143bff1da2696835a256aa074c2862462d853a442e539a1c7f6321d","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=gpt-5-pro-2025-10-06; createdAt=2025-11-06T22:48:20Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_740eca306e42940f1961dbec","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-12T20:32:51.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-12T20:32:51.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"619c335b5e4ea1dfc02d1e673b6c7ced17e23ea4ac6279c920ea54278e917eea","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5578947368421052","leaderboard_private_problem_count":285,"model_release_date":"2025-10-07","public_example_count":10,"stderr":2.946997752822398},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"TFDQUeRKbWiEodRQRrKdpy","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTFDQUeRKbWiEodRQRrKdpy.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/TFDQUeRKbWiEodRQRrKdpy.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5 Pro","thinking":null,"upstream_model_id":"gpt-5-pro-2025-10-06_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"2a925f27611cfc4f072e4519a4b360a41eb05d1f9ed8b043fadc5380bcef4dcf","result":{"confidence_high":61.56558927974242,"confidence_low":50.01335808867862,"sample_count":285,"score":55.78947368421052,"score_display":"55.8","source_stated_rank":41},"run_fingerprint":"b79486e6d91b091ffd1db93a9a28e526656b895e1aedd8c2f9cebcfe2b2a25d5","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_41d7f9a89ca5784ec9ee7cc9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cba220f657573fc2c9a4c3d74187cf22c121889bc3460dc8cf9c58a13d2d43fc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":7.1432,"model_release_date":"2025-10-06T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"","model_type":"CoT","upstream_model_id":"gpt-5-pro-2025-10-06"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f84bb880e7ff569fc85493f3c50e51c08b6b89744f667f2c5c9bfea40be74076","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.33,"score_display":"18","source_stated_rank":131},"run_fingerprint":"968ec462306722de98cd6e2375543c033363933e33d1fcbfebfa9b2ae071c2d7","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4d96b3482cc8ff6458efef4f","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"65a6cba10a9244d56ea615d15f5e642c720d2c8d5ec3abaec52e58ba72562ef3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"7.1432","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-10-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 Pro","source_effort":null,"source_model_version":"gpt-5-pro-2025-10-06_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"685979b80cea102c99210e6975314cbf3f10b4270f7945e9cd29d8a382a2fb4c","result":{"confidence_high":22.654112,"confidence_low":14.674659,"sample_count":360,"score":18.33,"score_display":"18.3","source_stated_rank":128},"run_fingerprint":"4302d40688f9a6419e53f03694d2893953a5f62a4c9258ddef4109687bb78564","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_61c8e4378a6fe57cfe14360b","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-12T20:32:51.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-12T20:32:51.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a5295de45e25c5537c56a78d9d04b42dd09a0d8effdfbbd373566801a5badd92","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.1951219512195122","leaderboard_private_problem_count":41,"model_release_date":"2025-10-07","public_example_count":2,"stderr":6.265967111543964},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"HfDahF4ZXTN6bjfDFwcHTU","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHfDahF4ZXTN6bjfDFwcHTU.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/HfDahF4ZXTN6bjfDFwcHTU.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5 Pro","thinking":null,"upstream_model_id":"gpt-5-pro-2025-10-06_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"05763c32286b6be45f39cff1a4055d9332f8efc4bbc86129e31fbde534813702","result":{"confidence_high":31.79349066057739,"confidence_low":7.23089958332505,"sample_count":41,"score":19.51219512195122,"score_display":"19.5","source_stated_rank":47},"run_fingerprint":"f9f19e6d9a49f9b6e6aae5a4d92d20714cd5a304177ec3ee590ecfdc96315d9b","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_41d7f9a89ca5784ec9ee7cc9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a795e0d0c96ed2e99ecc57c4c5b83cb2da2c1276f2685bd176b126cf4ab23493","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-10-07","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5-pro-2025-10-06_high","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"ad655f65aa1b31d2d9d63a2310f2c01fa4fafbf17fbbec5b8e4ad61970f0330b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.6,"score_display":"61.6","source_stated_rank":25},"run_fingerprint":"a6cc732952b5dd1b8ee4e290aa970d228c4b59786e3f0abba07a25b9ed790050","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_41d7f9a89ca5784ec9ee7cc9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fcad0844ee9e5a6fe3577a9b08248434fce497f39937222c7fa41dc5d3ab4406","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-10-07","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5-pro-2025-10-06_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"01eaa4fb8fc3f2ce41d0e9aa6c67643ca46d79c642e3183323ce016efd33e260","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.6,"score_display":"61.6","source_stated_rank":25},"run_fingerprint":"93f1b9b5a887106528eb1f8ff7a54d32ceb366d0c285d70314b458de0dc4cd18","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3640ae1b87e27096e2fff499","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2940eabed1846732eb77506206d59520065d4c6023a7390d965eaf45a2299278","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-10-07","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 Pro","source_effort":null,"source_model_version":"gpt-5-pro-2025-10-06_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0710fb92f76c48f97b9299196eb91d3e8791c75043e363117df465fda34c7b5a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.19999999999999,"score_display":"70.2","source_stated_rank":135},"run_fingerprint":"4c575e1933d88b1cf47ea7eec49e25ffac03a392830689149fddded034345476","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_41d7f9a89ca5784ec9ee7cc9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"566b791a912d0b35001ced66a90d348197302bc153247eca832a9c3cfe91f9ae","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.7785","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-10-07","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 Pro","source_effort":null,"source_model_version":"gpt-5-pro-2025-10-06_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"8b7288a3409bb1c0ee923cea2a55a8a713e6d4b70a77d2857a107379503e3716","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.17,"score_display":"70.2","source_stated_rank":136},"run_fingerprint":"633afd0ce0c6c341b75fe7462656110cafd3487caba8a8691803b95baf12492d","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0dcc5756dee51b913015e25","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"84147636921c7f71849bf242a72e4dea61f7c06134a3bee1c3ce50c229901b1e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-10-07","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-pro","name":"GPT 5 Pro","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5-pro-2025-10-06_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"8950df63a272b779e4e72819ce872302ddaf277332626f77c0d5aecb854e2c31","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.39,"score_display":"60.4","source_stated_rank":50},"run_fingerprint":"563876cd498d96d74572912e0e480601c6a830902e2e3f1cd4be6c3e6edde619","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_41d7f9a89ca5784ec9ee7cc9","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4d6ada725b3f49ddd8471e7a53e80c715cf23ae4f255431a16222af78425e302","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-15"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Inkling-Small; model release: 2026-07-15","source_accessibility":"Open weights (unrestricted)","source_model_name":"Inkling-Small","source_model_release_date":"2026-07-15","source_model_versions":["inkling-small","inkling-small_xhigh"],"source_reasoning_efforts":["xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8d35a0dfbfe0677ee64f8e41e953ad8522cd90415d18bed97f2d64faaf0c3cca","result":{"confidence_high":152.05,"confidence_low":147.51,"sample_count":null,"score":150.16,"score_display":"150.16","source_stated_rank":48},"run_fingerprint":"a764408882e4dd3a5b401ea81a1f023a079ebbf567d85e0cc09f8be1291875c5","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_16db478ee7ae220f4d066083","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"33ed002b671e15036c3db71e95b748f0d2a1544a7b4c5d6db42b3bb92bd39af4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-30","source_locator":"Leaderboard models table · Inkling Small · intelligenceIndex"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling-small","upstream_model_label":"Inkling Small"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"547889e6ab1ca471edc7f2c21cc6997b5053449ac6613476bd6e8c9221715405","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.658258,"score_display":"25.7","source_stated_rank":null},"run_fingerprint":"89b89581ec5b5b3d3e2bfea0b16f5b87f3655360417d299e091d3339461d34b2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling Small · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20eb51e9f88cfd04e5ce2e9d","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"12d34f7989caaf7bc242204a4ee768817b203b99947cd63ea35f8cf6f01b7299","metric_details":{"license":"Apache 2.0","variance":6.243478690563019},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1410.1386236986993,"confidence_low":1400.3439177229839,"sample_count":27070,"score":1405.2412707108415,"score_display":"1405","source_stated_rank":152},"run_fingerprint":"17127844a5442821809a8d3273b13f248916d4405dd61248a96f708c734a64af","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9afb89ae1ce26b5639d73387","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"858ff185253c386f6517215ce906fd46c20d2d1df95b097ba7cf56988f760ca8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"text-only","notes":"text only","num_parameters":265956439090,"pull_request":null,"result_file_url":"https://huggingface.co/thinkingmachines/Inkling-Small/resolve/main/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/thinkingmachines/Inkling-Small"}},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.6,"score_display":"31.6","source_stated_rank":37},"run_fingerprint":"7af47a41e030ef07ea6084955bcd5e9a209ea5dd7319980adc77bb706d0e41af","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d2b3708222c5ac4dd0dbf632","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b157086bd15d6d68282584b13de495f506b9c46c326b35d1b8f97e413982a107","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-30","source_locator":"Leaderboard models table · Inkling Small · hle"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling-small","upstream_model_label":"Inkling Small"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"94afe38ea1508ae00577d3046237f549ef753a0f57a68a26dc2faef1f04eccac","result":{"confidence_high":35.334705,"confidence_low":31.360357,"sample_count":2158,"score":33.317887,"score_display":"33.3","source_stated_rank":null},"run_fingerprint":"c1b1a2deb9d4929fa8090398b36a12e131923fdb1bc9315f15e634daef158ebd","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling Small · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20eb51e9f88cfd04e5ce2e9d","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:36:17.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:36:17.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"353edff227e3e6b447c4ddd2509e496096f70b6baa18eef483d7d12634204f3d","metric_details":{"best_score_across_scorers":"0.191","model_release_date":"2026-07-15","stderr":1.2436787112179482},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"eJfbDKqeg9kNXFoxpeV7HG","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeJfbDKqeg9kNXFoxpeV7HG.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/eJfbDKqeg9kNXFoxpeV7HG.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"233a1f4e148821f06ce49380fe8bc1814d008506ed8870d14305ed38a37e8dc6","result":{"confidence_high":21.53761027398718,"confidence_low":16.662389726012822,"sample_count":1000,"score":19.1,"score_display":"19.1","source_stated_rank":74},"run_fingerprint":"ebede7b686dafb63a6427618fce82a8652dfbb8e9b1bc726e0f0d1983c13c802","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_11897b7e05ae3b489552bf1b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0524d125399513937f6f3c20aa2fe73f5cba13a094cc1c67ec4e939998c44e8d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.020022,"date_is_proxy":true,"latency_seconds":259.758,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.036},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":79.798,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":262000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"0.99","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling-small","zero_shot_accuracy":87.374},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"dbf5ef093a889cad2ec792e81faceefc6e46f96eeb294afaf99e882e077d388c","result":{"confidence_high":87.57656,"confidence_low":79.59544,"sample_count":396,"score":83.586,"score_display":"83.59","source_stated_rank":57},"run_fingerprint":"80049841618ee87048cbb1296e0f8182a6250c3779b6e8b2bef7c41185cf7cac","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e7221fcbe0df2d3f97b361b4","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7b15099f322496244164485359acb7825e9b512f10b075a691fbdf9709b1f97e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-30","source_locator":"Leaderboard models table · Inkling Small · gpqa"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling-small","upstream_model_label":"Inkling Small"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a6b3f9622ccd7714960268ad6528c1acb59b7b0b1c38202680fe0753a2142629","result":{"confidence_high":93.039607,"confidence_low":84.446897,"sample_count":198,"score":89.494949,"score_display":"89.5","source_stated_rank":null},"run_fingerprint":"040f7249e2ac2aafcc6a188e35abedd44776c69690ac523e8b1a886eca8d3a13","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling Small · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20eb51e9f88cfd04e5ce2e9d","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14T14:19:47.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T14:19:47.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b28b1856929d63a0a52db8b33e7e01d82ab1d0c2cf77b560d1605900b3589b74","metric_details":{"best_score_across_scorers":"0.88510101010101","model_release_date":"2026-07-15","stderr":1.9100012831823399},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jTHw5SET8usd7emYPjEDjS","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"07e1ad1b3eb3a685522b283a86fc4e9a8631b21b556e6b7120b426cba587c0ba","result":{"confidence_high":92.25370352513839,"confidence_low":84.76649849506363,"sample_count":null,"score":88.51010101010101,"score_display":"88.5","source_stated_rank":57},"run_fingerprint":"f398a54b219344445a9d806a1583c44cc49f61ebaa2896c4b1b1ce560622ded1","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_11897b7e05ae3b489552bf1b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d3503d3185f964b2f53eb6d7303f38db34dd6c784168aa1907c4065895c95422","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.005706,"date_is_proxy":true,"latency_seconds":71.685,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.346},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":262000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling-small"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7040073415ab9c61c932de4142e981f3cb1733088e65569327f3d0ed604d7a66","result":{"confidence_high":86.24416000000001,"confidence_low":84.88784,"sample_count":null,"score":85.566,"score_display":"85.6","source_stated_rank":58},"run_fingerprint":"5faa66897b2acc76977745c4146fc294e98f5487f09ac2339d13f877b83e78b7","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4afb8d94c07f2a4577e683fa","provider_config":{"source_id":"vals-mmlu-pro","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":52,"at":"2026-08-14T14:19:47.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T14:19:47.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8852b06d020ffbef3ff1312e7b96f35f96a6bf84dd2973519da21f4205be2d58","metric_details":{"best_score_across_scorers":"0.9","model_release_date":"2026-07-15","stderr":3.964807305493795},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"53BDFebxiT9KGeP5FRt6bJ","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5ba7da82e26519352aaaa2731865202ebb0ad36f685e72035b623886ce822905","result":{"confidence_high":97.77102231876783,"confidence_low":82.22897768123217,"sample_count":45,"score":90.0,"score_display":"90.0","source_stated_rank":69},"run_fingerprint":"9b0c8b02bb42b130a58b9666a6a105210a0a0b05790043c074534aa2e9ea68f1","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_11897b7e05ae3b489552bf1b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":51,"at":"2026-08-14T19:59:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T19:59:09.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ed4b1089ca40d997ae14b730a782a1ecf7806484dc843f1f089c1bcff0118bc0","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.4631578947368421","leaderboard_private_problem_count":285,"model_release_date":"2026-07-15","public_example_count":10,"stderr":2.958888847874604},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"B6bZbqickq8o2fQcgDmzMo","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FB6bZbqickq8o2fQcgDmzMo.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/B6bZbqickq8o2fQcgDmzMo.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Inkling Small (xhigh)","thinking":null,"upstream_model_id":"inkling-small_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"1175a744640d4a5c8c5837b22e5668b81b7454a29aca4cbd40a878a02f6aff48","result":{"confidence_high":52.11521161551843,"confidence_low":40.51636733184999,"sample_count":285,"score":46.31578947368421,"score_display":"46.3","source_stated_rank":50},"run_fingerprint":"21d8117932fbb78d83f008fdfe09e67b4c766b528f7fa7a0aa77c64bddfbfe9a","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_11897b7e05ae3b489552bf1b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c5f319a3d30279f65ed7c022560ef4ca5cc4c71b0a03bad54c0a14505e922a76","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.23255201249999993,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/thinky-inkling-small"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"thinky-inkling-small","model_type":"CoT","upstream_model_id":"thinky-inkling-small-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f1c08ad01e5310c4d0aa22562f56e8ff60f51fd82895da2534adea920bcc71c5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.138888888888886,"score_display":"40","source_stated_rank":105},"run_fingerprint":"22e8d8bd6bbc1c528f1ff954bc20586a1d97d6e17f1ea9292150fe4e790d3571","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_11897b7e05ae3b489552bf1b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9d06c9838ec86760f7b4108980af6ee75c6226a6a0e1aafe38bcd1b83570c47e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.132909555,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/thinky-inkling-small"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"thinky-inkling-small","model_type":"CoT","upstream_model_id":"thinky-inkling-small-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1fb21dce9ff6217a62174327f537edf68718c19528a71ac3d21f0135ea9b981e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.05555555555556,"score_display":"33","source_stated_rank":114},"run_fingerprint":"36b35e7dcc3a04b5821e24ed84d7f529e9843b4a7455a3d2097a82a941ecc593","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f111864f5284f5b11c4dd036","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"91b7ab1606a560e73b655693c87e47ba6e452ca5005515c0c33eab66a02dd80f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.06378986500000002,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/thinky-inkling-small"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"thinky-inkling-small","model_type":"CoT","upstream_model_id":"thinky-inkling-small-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"174c0283011fdd2ba58c9e8e9b09649197b42c43480630b3ea320526cfee66b3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.611111111111112,"score_display":"14","source_stated_rank":136},"run_fingerprint":"007ddf9e5eefc3180991a200d82f4f07497ef4a909ecb5defcb25ae66c142275","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_0236b3bc9595971330a01096","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2f1bcf540253f33f18bf01568ed31b0e90533c6b7505a536469ff1835ead7e61","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.05589815499999999,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/thinky-inkling-small"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"thinky-inkling-small","model_type":"CoT","upstream_model_id":"thinky-inkling-small-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"29cdf5aac48fe47ce62c255fedfa599011c6d58fec5a27d90b7661c7c9118934","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4.861111111111112,"score_display":"5","source_stated_rank":164},"run_fingerprint":"8946807b2d7761cd394b7473121b710bb21cdd3058f03570d3bf619014a19d17","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_7685287e443c2d6d60d7fb92","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bf761d728a85c36d693f4d8a8a417383ae7ff3f9bba1a315f7fa1ee8c327460e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.05303395499999999,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/thinky-inkling-small"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"thinky-inkling-small","model_type":"CoT","upstream_model_id":"thinky-inkling-small-minimal"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5db72b44a351ea79f644a83758aca001c116e12e2e533e1417f5be6c6cecab7a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4.027777777777778,"score_display":"4","source_stated_rank":177},"run_fingerprint":"0556d17fe1e46c1640acc3b1adeb3b2c4abd3773ef8bc229f6f0f3d23c6ef9a2","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_64c0a7364600fc053b9711f9","provider_config":{"source_label":"minimal"},"provider_mode_key":"Minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"45590aa63ef9f288c76a952eda71da981f09fb91562b7974d975799d56e09672","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.010745875000000005,"model_release_date":"2026-07-30T00:00:00.000Z","results_url":"https://arcprize.org/results/thinky-inkling-small"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"thinky-inkling-small","model_type":"CoT","upstream_model_id":"thinky-inkling-small-none"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9616b9ea212e08c7b3bbf53e3e5f8fd66f72087663bd26e844a73bfcc77f1c96","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.0,"score_display":"0","source_stated_rank":239},"run_fingerprint":"aa747a27d8ea4330d4779afe1bc7969f0f5e82ee64ba42429b9808a5cdd38696","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_1926366c4272ed8031e24195","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e8e98b059ac57c72abe381ffc66c87a5d7b39c8179f315f7ba8918b556c5d142","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (XHigh)","source_effort":null,"source_model_version":"inkling-small_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"13b90a9e65a59b335a472c60f3e11b34bc96a90628cf987ce60c8b34ffda251b","result":{"confidence_high":45.280886,"confidence_low":35.205128,"sample_count":360,"score":40.138888888888886,"score_display":"40.1","source_stated_rank":106},"run_fingerprint":"2ab2b9d9f3e4744968d8c850613b5aefde81c7b3a610ef488e204e7ac3a74c57","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_11897b7e05ae3b489552bf1b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e56098f5823531deb2c771d200d266677871cad5ed39398c8177de4437fba587","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (High)","source_effort":null,"source_model_version":"inkling-small_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0ae6fedc6eed392818b135ff38b795f5938374e33df19f2d381aca693db7faf5","result":{"confidence_high":38.071469,"confidence_low":28.397456,"sample_count":360,"score":33.05555555555556,"score_display":"33.1","source_stated_rank":114},"run_fingerprint":"63e180b0653a880ca1ef8664fc25cb825a980a9e511dad330cb8db7a1c61b076","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f111864f5284f5b11c4dd036","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"206cc6f364c796da906a73e6760f5612913d61659590c2b6414fd7e35659c3e8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (Medium)","source_effort":null,"source_model_version":"inkling-small_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"18fd96cb026d589a12ffc9d6c1df53e812d26d055791a8acafdaebb2c6aa2774","result":{"confidence_high":17.539718,"confidence_low":10.450924,"sample_count":360,"score":13.611111111111112,"score_display":"13.6","source_stated_rank":133},"run_fingerprint":"29b7097635a1011358fbdfe00f597da7a215a270b7f8c02cef943fa8cbe36c6e","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_0236b3bc9595971330a01096","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d3d7a6805ac66fa5be5a9b22513a886c9fe28bc794bfd3b7aa46a3171f3bbb57","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (Low)","source_effort":null,"source_model_version":"inkling-small_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"9903b594b9d894a4ca16a67af1f5046233ca400bb4d65a8cdfa1a85f1e86bb26","result":{"confidence_high":7.598287,"confidence_low":3.077128,"sample_count":360,"score":4.861111111111112,"score_display":"4.9","source_stated_rank":160},"run_fingerprint":"50bca164c771ead1767ccde1c46e151e1334460a058def459588d4878ae4c271","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_7685287e443c2d6d60d7fb92","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"654da5f4babc9ce1c67a4064650a54ed4b384e18d6711579f4df42854d8a1978","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":11,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (Minimal)","source_effort":null,"source_model_version":"inkling-small_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e48c51fc13141a145637bbd91d3e39cbc0345d849670ece5e164f32100001d6f","result":{"confidence_high":6.590918,"confidence_low":2.435428,"sample_count":360,"score":4.027777777777778,"score_display":"4.0","source_stated_rank":173},"run_fingerprint":"6d13db8ad954f439afd5f9dc831236f2db7a4cfd23822767feca9a2d95846ad7","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_64c0a7364600fc053b9711f9","provider_config":{"source_label":"minimal"},"provider_mode_key":"Minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1e587b5455c596780c520fc110b19be9b1d85aba2a9cd2664b21ec98e4ad6686","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":12,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (None)","source_effort":null,"source_model_version":"inkling-small_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"242a1893537924f51e518bdd98adcc339b2188c5b7cd10b788e17c08728a022f","result":{"confidence_high":1.055844,"confidence_low":0.0,"sample_count":360,"score":0.0,"score_display":"0.0","source_stated_rank":230},"run_fingerprint":"bdd002ad0e396ce87ab5a91a1c209def9a45997d32515716948f519cc645b521","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_1926366c4272ed8031e24195","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":51,"at":"2026-08-14T19:59:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T19:59:09.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"70799fc3326e907d480330d5ccb2cf7e1aaae39d485eeba61e227c48cae8d90d","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.17073170731707318","leaderboard_private_problem_count":41,"model_release_date":"2026-07-15","public_example_count":2,"stderr":5.949419959829496},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"DVD8G8DHNyzkowBzK7JvMt","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDVD8G8DHNyzkowBzK7JvMt.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/DVD8G8DHNyzkowBzK7JvMt.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Inkling Small (xhigh)","thinking":null,"upstream_model_id":"inkling-small_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"8a4e42c5a1b8e6bf5f6c5dbb9c7133e83186afc175e7a112a74935b3030c4f51","result":{"confidence_high":28.734033852973127,"confidence_low":5.412307610441507,"sample_count":41,"score":17.073170731707318,"score_display":"17.1","source_stated_rank":48},"run_fingerprint":"954ac98e4e04c6279a403278b40dc2e0ef3489449a7e2b3c5d97bc25e13e85a9","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_11897b7e05ae3b489552bf1b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"89b31ece0e3df9fe6237bfcc11a7cd5430279e82ebf7ce67ebf84580d7c3476b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-30","source_locator":"Leaderboard models table · Inkling Small · mmmuPro"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling-small","upstream_model_label":"Inkling Small"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9334bb911d9c73b7a042ac6537bfdebe02a7df60f2a3d793f7e8bb0072a171cf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.046243,"score_display":"74.0","source_stated_rank":null},"run_fingerprint":"00e04fd086d29ec22b749aba14ad64d29a50f6db38c35f7c49c619bb3fbfb3f4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling Small · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20eb51e9f88cfd04e5ce2e9d","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"39b74e55fbbdfc91cc08707a057c43c14e41f51a4f13a70c89c59c2bef8464dd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small","source_effort":null,"source_model_version":"inkling-small","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"1e42df84b03ef7e5a60c4a5135689756547a46da324be7916de072cad63a0d16","result":{"confidence_high":17.029252,"confidence_low":3.824544,"sample_count":71,"score":8.28571428571429,"score_display":"8.3","source_stated_rank":100},"run_fingerprint":"6802cd6c7f72cb0b86cc25793f54b6b603eff0ac3cc901854c00efeeebe8cdd5","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_990fa01dbf65734127415da5","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8cc62989fbb13d0007479ccdbd4688fc5cd3ecab64a041a36877770cbecab9e2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-30","source_locator":"Leaderboard models table · Inkling Small · critpt"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling-small","upstream_model_label":"Inkling Small"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b1c66671d1feed595e84b1051ed08b9fb128f518a57c2eb80fe7f9472c0e32cd","result":{"confidence_high":17.107536,"confidence_low":3.804253,"sample_count":70,"score":8.285714,"score_display":"8.3","source_stated_rank":null},"run_fingerprint":"7b73b7414eb5451272bc0bac65831f2d015a99b28659a6874249aab05ebfa319","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling Small · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20eb51e9f88cfd04e5ce2e9d","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":52,"at":"2026-08-14T14:19:47.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-14T14:19:47.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a65ff5a18800c6bc93552550273074e08bff53ac17f87045c076d9ee52fb29ad","metric_details":{"best_score_across_scorers":"0.18","model_release_date":"2026-07-15","stderr":3.8612291966536914},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"YyEeZDLsvCT5LdNjR9cUrg","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8a4f78fe674f02aff2f9d7c2ff6ae703c5cc84a6022ce988c9291e7f52400beb","result":{"confidence_high":25.568009225441234,"confidence_low":10.431990774558766,"sample_count":100,"score":18.0,"score_display":"18.0","source_stated_rank":101},"run_fingerprint":"3130692d12c11fc7a4762e6b2f57dd447aaef00dcee0eeb797c9362a4644cf3d","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_11897b7e05ae3b489552bf1b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a27fcf5691de355a492957c7630e67a71bbe671aebdea35ce73e87444d96ad57","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (XHigh)","source_effort":null,"source_model_version":"inkling-small_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b23c7b89a7b59a9f38b93fdb7965b185db7bf5d22ddf0951429c283f83756fae","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.0,"score_display":"84.0","source_stated_rank":106},"run_fingerprint":"9a16f626c1b81018679550cded95c0db630fffe3a2800c627361086ffa3ea743","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_11897b7e05ae3b489552bf1b","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b08a6e01fd9aedb301659a0550d4abe1b4bfb42ffc5af57f26637a6603c83785","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (High)","source_effort":null,"source_model_version":"inkling-small_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e209f88e512c2df10ade936d966165b9c0dd8c82285baecc427af96e320a6975","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.0,"score_display":"78.0","source_stated_rank":116},"run_fingerprint":"ac97dc829474adccf0bde4c924633b7d5effbd08f542335fdfa8fe7ff303cb1a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f111864f5284f5b11c4dd036","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5c5a841ca6e620a64fc7538ab48c19f6f613c1a87bf1b494cd3f7ce9316b9157","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (Medium)","source_effort":null,"source_model_version":"inkling-small_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"021f78a85566b14a47f59350c396952fa540f02ac23212316592b081183e8521","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.0,"score_display":"67.0","source_stated_rank":138},"run_fingerprint":"71e41448ac4d233e8f965e7f84591ed7d2f37e2fd79107fb1fda6c8c66d3488b","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_0236b3bc9595971330a01096","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c731cb55156db38054cf4458134b39a09a1fe3f9fba0aaa09df45a4a01ee7f10","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (Low)","source_effort":null,"source_model_version":"inkling-small_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f4af99359b0840bed44d40dcce005d0d63caa7cdf34d57cadf3abe1d9f6fece0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.5,"score_display":"52.5","source_stated_rank":165},"run_fingerprint":"4076d5a9cf0c927cf94a796ebaf54526af7d28549901ee021db19c6a35ac1b5f","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_7685287e443c2d6d60d7fb92","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9d62df2fd72336c15726388dfe12cac123bbbdffc07a5b8ac4b7d3c3eebb25b3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (Minimal)","source_effort":null,"source_model_version":"inkling-small_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"97a446a0dacfcf53f111d9dacc6f863e0cd0c723e2d68b421f70850367f6e9cb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.5,"score_display":"48.5","source_stated_rank":169},"run_fingerprint":"44304360cb2d3da8b3d04f86783863881898fea3fdde8073fdb0b843de2c7b18","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_64c0a7364600fc053b9711f9","provider_config":{"source_label":"minimal"},"provider_mode_key":"Minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7f27223ed76c009da6695702196cf60f21e697054c10de91dec0fda269bf5535","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small (None)","source_effort":null,"source_model_version":"inkling-small_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"65da23662381c8596232942bf8e4abf86045946b29099a7a6450a09f6dd1a66e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5.5,"score_display":"5.5","source_stated_rank":257},"run_fingerprint":"326a8ae7a0056bdb3eb1c3a54c16b43e86c24a12d65e4f67cc812e9bdcd8d251","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_1926366c4272ed8031e24195","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c10c13cea6bb442b64bde7b96504b3075712a91500190a688a53b1f6fdf9dd0e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.13413,"date_is_proxy":true,"latency_seconds":245.037,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.712},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":262000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling-small"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"92d468c2a11c8d03ec2627cb84e6cbd9582dacf0da68e1b0213018d16c07d14c","result":{"confidence_high":85.55552,"confidence_low":78.84448,"sample_count":500,"score":82.2,"score_display":"82.2","source_stated_rank":20},"run_fingerprint":"5c6e7adbb86ba0c792434b7a02b8aee4b3773a9c70c7f82482a7df5229ce3aea","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_703781bfe8a825dd1e4f60a3","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_910cf16f9a90b2398ada00a7","split_or_window":"vendor-public-protocol-undisclosed","unit":"percent","version":"SWE-bench Pro (public) · Thinking Machines provider table"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":67,"at":"2026-07-30","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:18Z","last_confirmed_at":"2026-10-05T12:33:36Z","observed_at":"2026-10-05T12:33:36Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-30","status":"fresh"},"measurement_id":"82527538182854fb85dd865da9d4ef66525fcd02632f94e82aac6b0be405458c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · Evaluations · Agentic (coding) · SWEBench Pro (public) row · Inkling-Small column; no SWE-Pro-specific footnote"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Thinking Machines Lab","evidence_verified":true,"freshness_proxy":true,"protocol":"Thinking Machines provider evaluation; SWE-Pro-specific harness and reasoning effort were not disclosed; the card's bash-only footnote applies only to SWE-bench Verified","reasoning_effort":"reported","result_published_at":"2026-07-30","source_content_hash_method":"raw_bytes","source_content_sha256":"2de45e1f0d8d387626aa063dccd54c1f519c21b2adec1c1fd597e451ae08d5db","source_published_at":"2026-07-30","tools":"not reported"},"run_count":null,"scaffold":"not reported","tools_allowed":"not reported"},"protocol_fingerprint":"7c4504f8a1fc135646a42de73251feef4da89651b686dd41ea3127d2231a3289","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.9,"score_display":"55.9","source_stated_rank":null},"run_fingerprint":"20047d04b78fb5b8ba10a3279ab71d99a42e251ae731182c008172730ba2c7d1","source":{"content_hash":"96dcbe4ed236fe4596e0626a1355fd424fbe2c4f66014d203cb1ccfc0aa4b3ca","fetched_at":"2026-10-05T12:33:36Z","first_fetched_at":"2026-09-03T09:21:20Z","homepage_url":"https://huggingface.co/thinkingmachines/Inkling-Small","id":"thinking-machines-inkling-small-card","last_fetched_at":"2026-10-05T12:33:36Z","license":"Apache-2.0 model card; aggregate factual scores with attribution","locator":"Immutable README · Evaluations · Agentic (coding) · SWEBench Pro (public) row · Inkling-Small column; no SWE-Pro-specific footnote","name":"Thinking Machines Lab · Inkling-Small revision-pinned Model Card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/thinkingmachines/Inkling-Small"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_83395fdd728ead3993ec3c23","provider_config":{"source_id":"thinking-machines-inkling-small-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3cff1707a24433fddb0d74ee585427b820cd374238ed1c97f7eb78ba9e8e3827","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.337871,"date_is_proxy":true,"latency_seconds":678.056,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.475},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":262000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling-small"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"160534fdd01c3a29bbbf1c7f46aa3deda7dd2de8be3ac4325114d6bd14d12eba","result":{"confidence_high":18.104,"confidence_low":0.5620000000000012,"sample_count":null,"score":9.333,"score_display":"9.3","source_stated_rank":39},"run_fingerprint":"3a800e3c9bdbd8efc69de86c74a99e13698918decf765e586d0e63ee171ed362","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dd301f51b56b79061d8b85c8","provider_config":{"source_id":"vals-ioi","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8712d705391fe0998e7392096b539ca583be494be6c2ab329daf5443c19bc27f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.020896,"date_is_proxy":true,"latency_seconds":302.324,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.992},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":262000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling-small"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"ab6b2e2ffc5da382e98657a7c241d8712069d649b9c2eef8444d9ee7a26383d0","result":{"confidence_high":87.87532,"confidence_low":83.98667999999999,"sample_count":null,"score":85.931,"score_display":"85.9","source_stated_rank":26},"run_fingerprint":"9c538d5a0e1c4cb25b0f5be53c24052cc33228227beb6c0c85852cf52c401abe","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e7bf88fa29d99292354422f8","provider_config":{"source_id":"vals-livecodebench","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d8b20b8ef8d364ff1ff733e1a363c4898eec5ee05de7a434a86e9617d3104159","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling Small","source_effort":null,"source_model_version":"inkling-small","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"f0903fc29e56ecd3275b78d6d34d7b354f54d7774c9a741193679cf523544919","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.7268518518519,"score_display":"48.7","source_stated_rank":101},"run_fingerprint":"31a531e911809c6cc6e3c726c44bd99201e72235b3c2f28df806cb2ba3b82637","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5ef7626ca399657d16aa77c6","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-06T00:23:09Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T00:23:09Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"308f942e3b7ee6cf7aff9dfc9cc4219b06199be0d9b98e32f78e3b506e9066ee","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-30","source_locator":"Leaderboard models table · Inkling Small · scicode"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling-small","upstream_model_label":"Inkling Small"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2e23d816c8c52b48e9c046effe2cacae5ff7ebb1d0f0cfa1304ef28af3ae43fe","result":{"confidence_high":55.393784,"confidence_low":43.920913,"sample_count":288,"score":49.652778,"score_display":"49.7","source_stated_rank":null},"run_fingerprint":"8fe084ac085b91d7a289acf72d25ab6781867275f317e67f98d199fa00fa5e40","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling Small · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20eb51e9f88cfd04e5ce2e9d","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"beb739582aba9a68710581daf254a1f8e07142ff7eba9faf3b4a9f557cb89a7a","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.432264,"date_is_proxy":true,"latency_seconds":2949.469,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.403},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":262000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling-small"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"c7219aaa2daf7c82ed09b372f572099d7dee1c280c0ffb96130aaefd6ac7ccb1","result":{"confidence_high":25.72188,"confidence_low":12.38212,"sample_count":null,"score":19.052,"score_display":"19.1","source_stated_rank":80},"run_fingerprint":"c70375906a695a41ae9ef2954e1c5ca32361514d9841430c00922ac4a055ede5","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1a5020ecdcfbe5f23617785d","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cfe69a56420b449f6d68cf665717fd9fdb4eabc32dd2fa9eb127365d3e0cc14e","metric_details":{"license":"Apache 2.0","variance":21.304921452903166},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"Inkling Small","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"da628121b1323e892cc1d7e50420538b5d3ab7894e089e16fdfd79700ec2160e","result":{"confidence_high":1417.2836901795165,"confidence_low":1399.190378880647,"sample_count":5421,"score":1408.2370345300817,"score_display":"1408","source_stated_rank":82},"run_fingerprint":"4c7e208bfec7ebf87df2d0c6f6c7a8207744aa9f47fd75d982d88163cf2c4e49","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_13830015e96667e4be7dbbf9","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9a23b484a5c4ed00fccc02f0691df55a4bed59403be116123cdcbfa7aebc62e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"inkling-small","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"708ff2c4f959ed9561295b5d8b76d056dbf3efac77812d8f38443636a9f0df8d","result":{"confidence_high":1415.95,"confidence_low":1397.21,"sample_count":5058,"score":1406.58,"score_display":"1406.58","source_stated_rank":81},"run_fingerprint":"374535eba1331d645555c95d8cf680c9c95475266c3c6949f35e5d7195138cc0","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_503e9b9396f3960487efebf1","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b0db840626088423c0084bfe488c0a1802701ae05e9d8a18c461334b418d1854","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.127848,"date_is_proxy":true,"latency_seconds":603.689,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.339},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":262000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling-small"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"6052ee1d81b625d7e8aef36d60fa5b36f576172821c7c7f31491d1a1edb68a47","result":{"confidence_high":59.64044,"confidence_low":50.47156,"sample_count":null,"score":55.056,"score_display":"55.1","source_stated_rank":49},"run_fingerprint":"2cdac487f608f170dff44ff4880fde55b8c40f35bf23c6cbfe763c91a0488cf4","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_629f3e62b174a633d2359fb4","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"649d0b12930f4df3befee65e00ebbd6fff1c3494c8c9fd21aa2d0545696f29d9","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Inkling Small · terminalbenchV21"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"inkling-small","upstream_model_label":"Inkling Small"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"c4af64e31e4641810f7827febaa0d238b8f6a8e7befbc7a90bba8c3cb72a720b","result":{"confidence_high":64.967783,"confidence_low":44.726148,"sample_count":89,"score":55.05618,"score_display":"55.1","source_stated_rank":null},"run_fingerprint":"c4496359c3f57313b567d2e296c525be1043a20103893c338e3e279cda332c7d","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling Small · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20eb51e9f88cfd04e5ce2e9d","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"df2a6fefed105c18bcd539ba7ce06c69814512ab79cc936d3ad94e2329981fdf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-30","source_locator":"Leaderboard models table · Inkling Small · tauBanking"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling-small","upstream_model_label":"Inkling Small"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"a6896f4e9e201c39fbe2f6f34a6228cf87121b3d09d82e5864851291589dd371","result":{"confidence_high":27.66538,"confidence_low":12.240374,"sample_count":97,"score":18.762887,"score_display":"18.8","source_stated_rank":null},"run_fingerprint":"37d9f0c05adb4ce78c5fbeb2507af2800de9b1f17de0d1f4f2679fdc6eccf80f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling Small · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20eb51e9f88cfd04e5ce2e9d","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":66,"at":"2026-07-30T19:17:22Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-30T19:17:22Z","status":"fresh"},"measurement_id":"9ae9a2eddc2e9fcc06b2dcf7343111fbd3817fb0e9b66b6f38e4d57433cf30f6","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.5,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=Inkling-small; createdAt=2026-07-30T19:17:22Z","source_row_created_at":"2026-07-30T19:17:22Z","task_pass_coverage_threshold_percent":75},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"Inkling-small","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"c022534a498d7881702a893838e73919209ad2312d2f8cfb1988d84fc84f59e5","result":{"confidence_high":81.7,"confidence_low":76.7,"sample_count":1000,"score":79.2,"score_display":"79.2","source_stated_rank":2},"run_fingerprint":"4a5e1401d4282b89a8992183f2c98f96ec648e57b6771e444a3b5747cb44e409","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=Inkling-small; createdAt=2026-07-30T19:17:22Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_318b9d51698a9143a85074a8","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"db751ce0699510925395023d3ee0282dc1d4cec98a6c4b1126051999694f6b76","metric_details":{"confidence_level":0.95,"license":"Apache 2.0","session_count":13273},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Inkling Small","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5753b01a54f53ebe0e81155a008f312c914282f6d7cd55aa4485a4531ebcb788","result":{"confidence_high":-0.08920547317873567,"confidence_low":-0.11732701403245874,"sample_count":503653,"score":-0.1032662436055972,"score_display":"-0.1033","source_stated_rank":47},"run_fingerprint":"6f953b8c8066a97893057f4401444dc6c80097b79135b66cd9aa16e0bdccf0f4","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9487927c8b4ab32b43e1f141","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"45f09453ea5ae6cca5934645c9f2e4ce7ae403a58110b9ac6dc7b4e53f9e0c74","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-30","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Inkling Small · gdpval"},"model":{"id":"thinking-machines/inkling-small","name":"Inkling Small","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"inkling-small","upstream_model_label":"Inkling Small"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"cdd76fd7f5aeaebf64b56ccb8e937b26da15668ec301321bb665f712c38409e9","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1108.58,"score_display":"1109","source_stated_rank":null},"run_fingerprint":"8a06839c2203c8fdcd64f6b0dca3347969c24d803d440da946e3774477d37fa6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling Small · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20eb51e9f88cfd04e5ce2e9d","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9115099471c6cfc9a233f0fb5c6b8e068590281221bf5b549fe1fbf24ab6960f","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-11-24"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Opus 4.5; model release: 2025-11-24","source_accessibility":"API access","source_model_name":"Claude Opus 4.5","source_model_release_date":"2025-11-24","source_model_versions":["claude-opus-4-5-20251101","claude-opus-4-5-20251101_128K","claude-opus-4-5-20251101_16K","claude-opus-4-5-20251101_32K","claude-opus-4-5-20251101_48K","claude-opus-4-5-20251101_64K","claude-opus-4-5-20251101_high","claude-opus-4-5-20251101_unknown"],"source_reasoning_efforts":["high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bb6b0f6e95ca4a9e261c006e408bb4d669260f5069956f00e7672074649fccf1","result":{"confidence_high":152.97,"confidence_low":147.87,"sample_count":null,"score":150.1,"score_display":"150.10","source_stated_rank":49},"run_fingerprint":"37cd8022df22f48e8105c187e7e2c332611f4d30edf769ab18fd127534eec00d","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5e834ff6a9d4d675c2c021a","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0fc9b53d004e24f0473e111942930e87c62fd353bb99187534c5623f527ae2f9","metric_details":{"license":"Proprietary","variance":3.831122291837142},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1477.4652903097438,"confidence_low":1469.79271615992,"sample_count":37448,"score":1473.6290032348318,"score_display":"1474","source_stated_rank":39},"run_fingerprint":"0aab1db0cda0d14dab91b60c7a8eebbd3a180f9c365a4f17b5aca3e1caa2f59d","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_130345efddb139a5f20d658e","provider_config":{"source_label":"high","token_budget":32000},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f94d11a9698e634132fc8e360bdcbc13ce38a93db5aa316f1c6e0ef61cacf1ea","metric_details":{"license":"Proprietary","variance":2.5611191160210995},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1472.906851044801,"confidence_low":1466.633595552128,"sample_count":72772,"score":1469.7702232984645,"score_display":"1470","source_stated_rank":47},"run_fingerprint":"ecd5fa5a366e8a5c2dc0d646e30c82d252788002ed672c2bb0b0a58139e86504","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5286c20d9fc122b7ede638a","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f436fdbdaefffd56c2be3e6279dce0421054cc8509914677ac5c7fc70e0d7e16","metric_details":{"category_scores":{"Agentic Coding":39.697,"Coding":79.654,"Data Analysis":74.44166666666666,"IF":62.54599999999999,"Language":81.26100000000001,"Mathematics":90.38900000000001,"Reasoning":80.0865}},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.58216666666667,"score_display":"72.58","source_stated_rank":49},"run_fingerprint":"be5cef8e0f730b6e3125da0b72691d8818c0b0849d3871dcef9631f56318d6d1","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_139bf317a1ccf46d0ad71c1b","provider_config":{"source_label":"high","token_budget":64000},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"token_budget","token_budget":64000},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":312,"at":"2025-11-26T20:25:03Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-26T20:25:03Z","status":"stale"},"measurement_id":"7d66eb98be54775d7fbe229b13ab161771a8e2a6aa4f53db095590cd1991f82e","metric_details":{"calibration_error":56.0,"confidence_interval_half_width":1.37,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=claude-opus-4-5-20251101; createdAt=2025-11-26T20:25:03Z","source_row_created_at":"2025-11-26T20:25:03Z"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"claude-opus-4-5-20251101","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e9d0dfe53d8358a8fc26520d4b752b04a85398c373669f8997dcf35862a47cfd","result":{"confidence_high":15.530000000000001,"confidence_low":12.79,"sample_count":2500,"score":14.16,"score_display":"14.16","source_stated_rank":23},"run_fingerprint":"b9e9a7dcf5297cfcc05e35b39694769adbd98c466850998b997aca6c6ddcdc0b","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=claude-opus-4-5-20251101; createdAt=2025-11-26T20:25:03Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7c52e44fa1806e202f8c3942","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":312,"at":"2025-11-26T20:23:24Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-26T20:23:24Z","status":"stale"},"measurement_id":"820e07bfafbe3b9defc7823751131ee200fdcf5ffa3e2ae357aa55f28bcdd8a8","metric_details":{"calibration_error":55.0,"confidence_interval_half_width":1.7,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=claude-opus-4-5-20251101-thinking; createdAt=2025-11-26T20:23:24Z","source_row_created_at":"2025-11-26T20:23:24Z"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"claude-opus-4-5-20251101-thinking","source_reasoning_label":"thinking-unspecified","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9f1c967306aeca8dd8b2f74e9242fe3633953264e57521b5f13ede2c6a0112ef","result":{"confidence_high":26.9,"confidence_low":23.5,"sample_count":2500,"score":25.2,"score_display":"25.20","source_stated_rank":10},"run_fingerprint":"4740fc267311a06e24a817dbb70191a1c00229cb5457ae778d44d6ff30e3ec1a","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=claude-opus-4-5-20251101-thinking; createdAt=2025-11-26T20:23:24Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7c52e44fa1806e202f8c3942","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":315,"at":"2025-11-24","basis":"evaluation_at","evaluated_at":"2025-11-24","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f43c12a3a47ce3bb29ded2305d6ea4b5ce55d6b338041268f22a8b8f27326c61","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/anthropic-claude-opus-4-5","source_model":"Opus 4.5"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4ca6989f3121b9467b4ce3a9665b220d0859c93c0e6a0f6dd6a6a20b107ac975","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.8,"score_display":"25.8","source_stated_rank":31},"run_fingerprint":"db538738ddd457bcf59e70b2333504344278e2547eb709777e79f97a607a52ba","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a913278e59364165e5f8c623","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":314,"at":"2025-11-25T00:00:00Z","basis":"evaluation_at","evaluated_at":"2025-11-25T00:00:00Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0e8b1bc29eb64f4a66919d527c0f8a855d8d34e139b8bfccaec50f515411dea1","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-opus-4-5-20251101","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"644c85525faccfd7fc3a26dacec1284e5908cc1abd3c0b92a30923675045f344","result":{"confidence_high":42.328430969901014,"confidence_low":35.679867785285715,"sample_count":null,"score":39.004149377593365,"score_display":"39.0","source_stated_rank":19},"run_fingerprint":"886e67f10835a322eb9fc691edc108c2c1096b08782f94076f8f4f2bb0282317","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a72b975b52d107a214a49b26","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f55ac8cb6e26e86f6cc720a9a85057ef4678812790d036eafcf5268531bf507d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.712758,"date_is_proxy":true,"latency_seconds":138.525,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.476},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","few_shot_accuracy":86.364,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":64000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101-thinking","zero_shot_accuracy":85.354},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"8517b7785167809f774180d9328fa040fd06fc6f9a8b732f4c038f1a64ec9952","result":{"confidence_high":90.71195999999999,"confidence_low":81.00604,"sample_count":396,"score":85.859,"score_display":"85.86","source_stated_rank":45},"run_fingerprint":"32cead34e319054a75f88ff96398eeebdbb6678d45707e1528b5cb64bdc3cf6d","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"55c7131f0c29e1a52268d8f2e4511a0b6f0baf6dbdbf31759e3d000da2c610ea","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.074932,"date_is_proxy":true,"latency_seconds":13.54,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.463},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","few_shot_accuracy":79.293,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":64000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101","zero_shot_accuracy":79.798},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"7c3e6180925582c498dae1967784d1f0bdced62c8474992997aa7bfe23fc66ca","result":{"confidence_high":84.37348,"confidence_low":74.71852000000001,"sample_count":396,"score":79.546,"score_display":"79.55","source_stated_rank":71},"run_fingerprint":"bd1c58a748c71540fc6e4aac9f52415bbd4b4122e30f57b01fc37432472d7b18","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":314,"at":"2025-11-24T19:57:20.151Z","basis":"evaluation_started_at","evaluated_at":"2025-11-24T19:57:20.151Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"2e7dd2145809ba85cb95d055529234588d9356d9aff119e23a7b3fc9dd9a9e45","metric_details":{"best_score_across_scorers":"0.8068181818181818","model_release_date":"2025-11-24","stderr":2.390947965376675},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mEkZaYpiurQ6TEMAALK3su","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmEkZaYpiurQ6TEMAALK3su.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mEkZaYpiurQ6TEMAALK3su.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"73c58e245ae8d8f74e4320bb94df75394a5b6f0c056b78ccfabb52711198e228","result":{"confidence_high":85.36807619395645,"confidence_low":75.99556016967989,"sample_count":null,"score":80.68181818181817,"score_display":"80.7","source_stated_rank":126},"run_fingerprint":"595b85ea404e318ea8c50d8011229a238a8b23ff10d45e379e0860f676bdc48e","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d5bcc1a44645bdf8314b6225","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ba777e573dc04dfaceb9043c0d248de6487f87916d2db5aa261b87e60e48cca7","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.215138,"date_is_proxy":true,"latency_seconds":38.92,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.381},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":64000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e0dc3633abd4d1076dccd76ebfe2f38591b5ab7529d7a8086d65f1dd6a8fcd4d","result":{"confidence_high":88.00676,"confidence_low":86.51324000000001,"sample_count":null,"score":87.26,"score_display":"87.3","source_stated_rank":31},"run_fingerprint":"75ea368e2d063e6fdc8ce180506cee631b1c90fc5ea1e875a174e46ea51c3149","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d50154e0893a39e2603e9d5a61aa2f8b4d7acf9c7189e748da1108a0dd37157c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.052065,"date_is_proxy":true,"latency_seconds":8.248,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.367},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":64000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8b2ff3be7335f39a3e666c7db8318344e2b8c1646dc55e4503e05e8c9e911647","result":{"confidence_high":86.30932,"confidence_low":84.87068000000001,"sample_count":null,"score":85.59,"score_display":"85.6","source_stated_rank":57},"run_fingerprint":"9cbb6b54b773c9e1e904e709761a5e87f8a4c324fd0fc30fb7e16ef13c8d3fc8","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":314,"at":"2025-11-24T20:47:49.364Z","basis":"evaluation_started_at","evaluated_at":"2025-11-24T20:47:49.364Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a411082c5f4fc027f0521dc5a07e8c4bac5c6a9c23e0e66263d2db22355e0636","metric_details":{"best_score_across_scorers":"0.48055555555555557","model_release_date":"2025-11-24","stderr":6.405336755004034},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"nSb2focDRMyPJXErk3MV94","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnSb2focDRMyPJXErk3MV94.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/nSb2focDRMyPJXErk3MV94.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"2fa30ce7c948cb2db42398112d6401b364afa08c3e7b7a939b5a2d4e2f252d45","result":{"confidence_high":60.610015595363464,"confidence_low":35.50109551574765,"sample_count":45,"score":48.05555555555556,"score_display":"48.1","source_stated_rank":191},"run_fingerprint":"e90da8dbf22eb8153c012d550d6247e40ce93527c27e8bf9f5464333a53bc193","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8163073ac76db4da90db1b36","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-11T01:55:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T01:55:25.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c77ba13d7a89b8b9588577cdfd2298ad2ef17c332749d8c6cce96ec2b1e08191","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.34385964912280703","leaderboard_private_problem_count":285,"model_release_date":"2025-11-24","public_example_count":10,"stderr":2.8185763989072585},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["32k thinking"],"epoch_id":"mS5EwCRZVpvUvJEehPUDZu","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmS5EwCRZVpvUvJEehPUDZu.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/mS5EwCRZVpvUvJEehPUDZu.eval","manual_evaluation":"","reasoning_token_budget":32000,"service_tier":null,"source_model_display_name":"Claude Opus 4.5 (32k thinking)","thinking":{"budget_tokens":32000,"type":"token_budget"},"upstream_model_id":"claude-opus-4-5-20251101_32K","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"25b36e77ff057a5430c68caa40fa10bc60073c75fd6fc13c93f34b7ddb269e8b","result":{"confidence_high":39.910374654138934,"confidence_low":28.861555170422477,"sample_count":285,"score":34.385964912280706,"score_display":"34.4","source_stated_rank":63},"run_fingerprint":"7bbe4eed2c715d0bac926d634adaf20f931e5eb12c4498c604ff4fe19434377c","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_597cba506275a64a67c0cb7d","provider_config":{"source_id":"epoch-frontiermath","source_label":"thinking-32k","token_budget":32000},"provider_mode_key":"thinking-32k","raw_label":"thinking-32k","reasoning_enabled":null,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"557caa0c4c2bfec1f1023f798dbb542c798ce2e683e3b11a3b4b0e6cc05c1a12","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.22","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.5 (Thinking, None)","source_effort":null,"source_model_version":"claude-opus-4-5-20251101","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"4b3084c25aa997f35d6830e86f41d21548d7307ed80b7e834cdb86431ea7ed8b","result":{"confidence_high":11.013984,"confidence_low":5.43757,"sample_count":360,"score":7.779999999999999,"score_display":"7.8","source_stated_rank":144},"run_fingerprint":"90b92bf71b4ec65db4f951fa8d255f3750fcd779e80959bdb8319c11c1bf1f1a","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_926bdc433ee7f4064ce2e951","provider_config":{"source_label":"adaptive"},"provider_mode_key":"Thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-11T01:55:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T01:55:25.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"6b75ab333bb9706d12994ceb180ac8ecd8313d33754c9b5bd16381d67b56f204","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.04878048780487805","leaderboard_private_problem_count":41,"model_release_date":"2025-11-24","public_example_count":2,"stderr":3.4059122057973035},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["32k thinking"],"epoch_id":"azk4theXxAsizmoi2HDD49","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fazk4theXxAsizmoi2HDD49.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/azk4theXxAsizmoi2HDD49.eval","manual_evaluation":"","reasoning_token_budget":32000,"service_tier":null,"source_model_display_name":"Claude Opus 4.5 (32k thinking)","thinking":{"budget_tokens":32000,"type":"token_budget"},"upstream_model_id":"claude-opus-4-5-20251101_32K","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"52c1ea0466e69627d2998ab353768b351763ae03145f03084de8f4c7c1fc4a31","result":{"confidence_high":11.553636703850518,"confidence_low":0.0,"sample_count":41,"score":4.878048780487805,"score_display":"4.9","source_stated_rank":58},"run_fingerprint":"c7b3c7ebc06aa1325693dcdc3a3349bb741962914a1294708a6402f334551fda","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_beecacca1fd5f3ea74568a4f","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"thinking-32k","token_budget":32000},"provider_mode_key":"thinking-32k","raw_label":"thinking-32k","reasoning_enabled":null,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9da905123fdb6274eb219a9c0d655885670caecd91612dbaeed232c01310c149","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.126986,"date_is_proxy":true,"latency_seconds":75.28,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.904},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":64000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"cb6ee1ccb6bdfadcc1461c154f85a2a5d9145a6213b003e3633e4bac83b7c3f3","result":{"confidence_high":84.71983999999999,"confidence_low":81.17616,"sample_count":null,"score":82.948,"score_display":"82.9","source_stated_rank":36},"run_fingerprint":"f53a335dc5ac9aa52bf9cb25fe03477eee09d15b08381545da08250906c50deb","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0fcebfa662dad6b2a65bae4215864ad23d787c2f42751b709f3f0b08ae8cc973","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.021549,"date_is_proxy":true,"latency_seconds":19.557,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.941},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":64000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1e3d65938d61e26b2064be53b9e750748f208aa791d82319c484edc858e520be","result":{"confidence_high":82.94236,"confidence_low":79.25364,"sample_count":null,"score":81.098,"score_display":"81.1","source_stated_rank":42},"run_fingerprint":"d38176c84ab7280b188924053a19dd822a6c825a867f74fcfbf5460fa4a5a0ff","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0c54d96c0bbcdd336a4931d4b6550d0dcf5f8d1e6b3f42b6e4153d573bf962e6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-5-20251101","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"bd4a0e71c7f9c39ccbd5b3fab0cdc3be9fce0b80743bd8d092a4ce230a25f1ec","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.0,"score_display":"62.0","source_stated_rank":22},"run_fingerprint":"1734b5810f8c41af56a49bc1ebf6b5f068e0a8ce411b99228c6f145a629757be","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_56fcc66f50d70a253236a8bf","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"592c45a44ecfabce28561ecdd208c3ba7aed2b1bce9f3fce9d69981b35f4fbde","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"b47c8d59337f7063ad0f4241d592199a85e880d96a02a053cf220fc74c6374dd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.0,"score_display":"62.0","source_stated_rank":22},"run_fingerprint":"4cc06fc2eaf09fedb19087b5e0fb614e79942e977d61e389ae8d2b32180365b8","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_56fcc66f50d70a253236a8bf","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:18:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:18:37.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"83791ee55ddaddb8aea41f5cb767d732076d27df0e1e9eb72667067e5d84d046","metric_details":{"best_score_across_scorers":"0.04","model_release_date":"2025-11-24","stderr":1.9694638556693238},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"2jTRxz7rW3B66a8smtz7Ya","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2jTRxz7rW3B66a8smtz7Ya.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/2jTRxz7rW3B66a8smtz7Ya.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"47c74cb7dfd9d5d5c906da848395e08d81bf7f338391ad3a50f70f30f41e09c9","result":{"confidence_high":7.8601491571118745,"confidence_low":0.13985084288812555,"sample_count":100,"score":4.0,"score_display":"4.0","source_stated_rank":169},"run_fingerprint":"3542249d3745431813a8a6433b90f426068f2bf173ece53d308629da88d42be0","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6eb479ddca0f4373f2bde9be","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"39a10011794f5032fd0260121524e4e43ac16a590c08f0b729d9fd8c2526fc49","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.13","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":""},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.5 (Thinking, None)","source_effort":null,"source_model_version":"claude-opus-4-5-20251101","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"bd170eff40b20afe8d63cbc3182879277ef4f6968b7f55e0284fbc36322a02dc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.0,"score_display":"40.0","source_stated_rank":181},"run_fingerprint":"906424654bf6e0038f432ab78cd65a1c04f82bf79eac8f78ac740511c4f297be","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_926bdc433ee7f4064ce2e951","provider_config":{"source_label":"adaptive"},"provider_mode_key":"Thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5c3c7ad2b994b5c5d17aef5d974fad0e64cb68385e2df42cada6964a97ad109e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.054837,"date_is_proxy":true,"latency_seconds":320.542,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.901},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":64000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101-thinking"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"a9b6508960081ae8b203f1ec4c31f3a8299c15b02d22f4c68d2a0a3c45e58d24","result":{"confidence_high":80.12596,"confidence_low":72.67404,"sample_count":500,"score":76.4,"score_display":"76.4","source_stated_rank":37},"run_fingerprint":"d15779812c553c294963f22276def00b5a20da9c9aa20273f3e4854e39e240c8","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":230,"at":"2026-02-17","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17","status":"stale"},"measurement_id":"783d023794346ba0a6272d3e6b18fbe621aa9386b2e8dffab22bbc51ced792fc","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_claude-4-5-opus-high","logs":null,"model_display":"Claude 4.5 Opus","model_org":"Anthropic","reasoning_effort":"high","resolved":76.8,"site":"https://www.anthropic.com/claude","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: claude-4-5-opus","Org: Anthropic","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_claude-4-5-opus-high/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"c45c34c307da1541a3737ae9f3c51a460d89e2bc063079a8398cfbf7f6312baa","result":{"confidence_high":80.287136,"confidence_low":72.904184,"sample_count":500,"score":76.8,"score_display":"76.8","source_stated_rank":5},"run_fingerprint":"06447ea72d9c9e4805108b43e2d276c599e316ce6de869ff5e470dc995688ebd","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":315,"at":"2025-11-24","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24","status":"stale"},"measurement_id":"42bf5937f2ec38978d98e15a9a36731339fa858d2cc488ad02d54c6be5025306","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":true,"date":"2025-11-24","folder":"20251124_mini-v1.16.0_claude-opus-4-5-20251101","logs":"s3://swe-bench-submissions/bash-only/20251124_mini-v1.16.0_claude-opus-4-5-20251101/logs","model_display":"Claude 4.5 Opus","model_org":"Anthropic","reasoning_effort":"medium","resolved":74.4,"site":"https://www.anthropic.com/news/claude-opus-4-5","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: claude-opus-4-5-20251101","Org: Anthropic","System: Attempts - 1","Mini: 1.16.0"],"trajs":"s3://swe-bench-submissions/bash-only/20251124_mini-v1.16.0_claude-opus-4-5-20251101/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"8541e3ee647322e0b2481d6832800d8445c16bc309e214834b97ad290d409a81","result":{"confidence_high":78.029292,"confidence_low":70.398626,"sample_count":500,"score":74.4,"score_display":"74.4","source_stated_rank":13},"run_fingerprint":"82fa17f23380d430174a28cc7d423099f9620851417a81de4b1b57471cbf4f58","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_8075e3f0ba34a2f39f22a5ae","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":241,"at":"2026-02-05T22:06:33.964Z","basis":"evaluation_started_at","evaluated_at":"2026-02-05T22:06:33.964Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c7ef54bf7288fa1f1d90158014f6b58f6eee1d508b42cd8223a079cd4395db62","metric_details":{"best_score_across_scorers":"0.7665289256198347","model_release_date":"2025-11-24","stderr":1.924895909142592},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mKQKxkpGDrFHBPBYR3mBrB","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmKQKxkpGDrFHBPBYR3mBrB.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mKQKxkpGDrFHBPBYR3mBrB.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"de98d26e7d61544c1747281dfe50ecf6e10e871293c8e4aee040b3991a74fd7c","result":{"confidence_high":80.42568854390295,"confidence_low":72.88009658006398,"sample_count":484,"score":76.65289256198346,"score_display":"76.7","source_stated_rank":9},"run_fingerprint":"b7cc01ce119e1d0ef8e2f7c6c195f857c928d23b447e8155bae1c503e3d0b4ad","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_db38d6f80d03299b3d1e9536","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":294,"at":"2025-12-15","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-15","status":"stale"},"measurement_id":"47a0d8f60b360beef5cc122958b22ef21b9aca4360b605605437eb8172311d36","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"live-SWE-agent","checked":false,"date":"2025-12-15","folder":"20251215_livesweagent_claude-opus-4-5","logs":"s3://swe-bench-submissions/verified/20251215_livesweagent_claude-opus-4-5/logs","model_display":"Claude 4.5 Opus","model_org":"Anthropic","reasoning_effort":"medium","resolved":79.2,"site":"https://github.com/OpenAutoCoder/live-swe-agent","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: claude-opus-4-5-20251101","Org: UIUC","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20251215_livesweagent_claude-opus-4-5/trajs"},"run_count":null,"scaffold":"live-SWE-agent","tools_allowed":null},"protocol_fingerprint":"521b667b8511041e953f82d314968671647b12c4bc8a475eee781b3b5eb843cc","result":{"confidence_high":82.528426,"confidence_low":75.426297,"sample_count":500,"score":79.2,"score_display":"79.2","source_stated_rank":1},"run_fingerprint":"a1675d24a7ee817ba7e950545a5733d77357937e3fb65496198bd37ffb354c2b","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_8075e3f0ba34a2f39f22a5ae","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":304,"at":"2025-12-05","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-05","status":"stale"},"measurement_id":"83827b8aec18d447821d99fd1b03ff1d8778ba43b19d222995f83a5b94203731","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"Sonar Foundation Agent","checked":false,"date":"2025-12-05","folder":"20251205_sonar-foundation-agent_claude-opus-4-5","logs":"s3://swe-bench-submissions/verified/20251205_sonar-foundation-agent_claude-opus-4-5/logs","model_display":"Claude 4.5 Opus","model_org":"Anthropic","reasoning_effort":null,"resolved":79.2,"site":"https://www.sonarsource.com","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: claude-opus-4-5","Org: Sonar","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20251205_sonar-foundation-agent_claude-opus-4-5/trajs"},"run_count":null,"scaffold":"Sonar Foundation Agent","tools_allowed":null},"protocol_fingerprint":"84dee480ad5b133b072dbee3821cfd5873c39f72ddbfcdcbd8881768f35ad183","result":{"confidence_high":82.528426,"confidence_low":75.426297,"sample_count":500,"score":79.2,"score_display":"79.2","source_stated_rank":1},"run_fingerprint":"69c6ad2193e9eab339fee4f9a9c01724759398afa83584d9fc02b808b46f935e","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ff965e909754406c2bffd4cb","provider_config":{"source_id":"official-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_1c18f4398e32b2e0fe7f7d86","split_or_window":"2025-12-01/2026-02-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":246,"at":"2026-02-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b0db892ecc6b19ca4db9caae151c345c3bf55c06a05cb6810cd972b8f021396c","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-11-24","pass_at_5":68.75,"potential_contamination":false,"sem":0.6249999999999991},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"Claude Opus 4.5__tools","window_from":"2025-12-01","window_status":"historical-post-release","window_to":"2026-02-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"55c67f29f9caeae2fb1a7af296decbc5ef7cf3d6983fdb72f33fcc65c4733416","result":{"confidence_high":54.766666666666666,"confidence_low":52.31666666666666,"sample_count":null,"score":53.541666666666664,"score_display":"53.54","source_stated_rank":null},"run_fingerprint":"14da9d4fd1eb38bd417024f7698503edefda5bf54acb32187b53beada1dd2408","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6d7d2d1817324164a6889f68","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":297,"at":"2025-12-11T23:31:01Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-11T23:31:01Z","status":"stale"},"measurement_id":"6764c28cd5bb83f60399f9b6c9d445ab2926a074ce1bf2657bd105d730edda7c","metric_details":{"confidence_interval_half_width":3.6,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=claude-opus-4-5-20251101; createdAt=2025-12-11T23:31:01Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2025-12-11T23:31:01Z"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-undisclosed-harness","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":false,"protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"claude-opus-4-5-20251101","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576"},"run_count":null,"scaffold":"submission-specific","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"1e1bc1d444f542d8d61f6a8454448ce9643515d4bf3b2a82b2f0b39c4b9e3e9e","result":{"confidence_high":49.49,"confidence_low":42.29,"sample_count":731,"score":45.89,"score_display":"45.9","source_stated_rank":5},"run_fingerprint":"7c17372862fd7f180ab1a4d15649c3053475ee38183dc9336b94befb89b9bcf9","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=claude-opus-4-5-20251101; createdAt=2025-12-11T23:31:01Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e6ff78ed13f94e3172beca19","provider_config":{"source_id":"scale-swe-bench-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-undisclosed-harness","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:04Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2815f116f16a121270ed5d83af2d75e15a53a9a3a1b83156f9c5a6452eb7a745","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":"GSO Leaderboard"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-5-20251101","source_row_date":null,"source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"72b1185f7df85085f15496c403080aebfabe615e1f401dd190b8f8a25bfe10a3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":26.5,"score_display":"26.5","source_stated_rank":11},"run_fingerprint":"11b1810a70d28701799931c44339e711de007f7be56c6bd249c90c65f135e5e9","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4c7c0b6e9b003d3102b8548e","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":311,"at":"2025-11-28","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-28","status":"stale"},"measurement_id":"908205891bc417962270bc9b7eb3997a9323de2afda2c4713b6f7c18e36bebe7","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":"GSO Leaderboard"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":"2025-11-28","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"75442f1d86216ee769712575ed531fc6cdcf8061aa1320cb42855ae700a815c6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":26.47,"score_display":"26.5","source_stated_rank":12},"run_fingerprint":"c30a0312afc366251547ce8378364506a2ac205875cda9cf7e07739387797838","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4c7c0b6e9b003d3102b8548e","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"268adb18d539afc38161502db507f1d598456dc3687724ba001cb45bef42c35d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.971,"date_is_proxy":true,"latency_seconds":164.427,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.037},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":64000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"26e734105e344e605a465916b3a8d4bb73b17bfc1a5c08969c7edd3bd185b400","result":{"confidence_high":85.70252,"confidence_low":81.63748,"sample_count":null,"score":83.67,"score_display":"83.7","source_stated_rank":47},"run_fingerprint":"747f61eb3685f29af628818880d36f8d9c4b0ce2f586de68f26142ee9f045a81","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ee863d21158827ac4be37e70ca8859f3d5316e1cc4d4660632d9e8e552b0c441","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.077323,"date_is_proxy":true,"latency_seconds":13.599,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.12},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":64000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"111b48eb73f67b7d56f867473c7f2a8409981ba257d2408d82ada0c48b5cd79c","result":{"confidence_high":77.2292,"confidence_low":72.8388,"sample_count":null,"score":75.034,"score_display":"75.0","source_stated_rank":81},"run_fingerprint":"ace41ab99add877b6a97caa85136ef5d468c591d687e1e6c9cbe2a2d05fb83ec","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"43bf51786820e0980fae981b48b80ba1aaab3fbc224a7fa6e0712019b80ec7c5","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":32.873283,"date_is_proxy":true,"latency_seconds":2283.142,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.159},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":"high","leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":64000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-5-20251101-thinking"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"c03c8eb4a6b18205dc79aeefb115706eb55ae839e1f1111d1a26115caed8d84b","result":{"confidence_high":26.82164,"confidence_low":14.43836,"sample_count":null,"score":20.63,"score_display":"20.6","source_stated_rank":74},"run_fingerprint":"4cc0e0ce03fe1845b941009b3801417309d898314c81a615b552e0f494097945","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fc128de63ae827b4c37e754023f66478f2d75b2b971212387f67b55d144d1a29","metric_details":{"license":"Proprietary","variance":17.7059630684882},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-4-5-20251101-high-32k","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1aaa2671c5ca9ab84ffd1693a4ed49c1f537f0c1efc32b2eb58724eb3d01b8cd","result":{"confidence_high":1501.5098823768167,"confidence_low":1485.01543143011,"sample_count":11034,"score":1493.2626569034633,"score_display":"1493","source_stated_rank":55},"run_fingerprint":"1f19b6e695ac2a2766b2d4efc3295a56bb55bc58c7f0a9b8ad757f71fef53467","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_130345efddb139a5f20d658e","provider_config":{"source_label":"high","token_budget":32000},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"79fbc3a49f6c5be0253f379242143a977848c6579fb80208c7e1680cc7d8bf83","metric_details":{"license":"Proprietary","variance":13.38157681981379},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-4-5-20251101","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"45cdee6dc12c247207e56f895af601dd4ce8cf46efbc3fae1a8dab0175a60637","result":{"confidence_high":1475.6304287067983,"confidence_low":1461.2910038495359,"sample_count":13427,"score":1468.4607162781672,"score_display":"1468","source_stated_rank":62},"run_fingerprint":"d5edce6eb77ef4a02869cdd59586c5a401e7dc4bac772995ab9723e7d77613c5","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b364a6aec4a9876629c9975c","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"6d84b207ff484bb607d861b8ca7f2df0155693b0663fc4705b027abebde2529e","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-5-20251101","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"d9afe47494c886e28ccd82d3695b892e8c8cfd44098c72b2d919d5bb3606d78b","result":{"confidence_high":1475.46,"confidence_low":1460.9,"sample_count":12899,"score":1468.18,"score_display":"1468.18","source_stated_rank":61},"run_fingerprint":"ab500dbb73e324491c367316f0a17bef23d2b85803814a4d3d23f26b332dc3af","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_757a5d6ab6fdf821ab369c8f","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":298,"at":"2025-12-11","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-11","status":"stale"},"measurement_id":"ebbfd876fb8c5ec59cf8810089517f35c90335dfba07309febdec49dd01a4f6e","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"standard_error_points":2.7,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-opus-4-5-20251101","source_row_date":"2025-12-11","source_scaffold":"Droid","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Droid","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"892ec7221dbaa52a5865b95edfab2d3923f7d3fbf17d2b20adb3cdeadcbf1352","result":{"confidence_high":68.392,"confidence_low":57.808,"sample_count":89,"score":63.1,"score_display":"63.1","source_stated_rank":39},"run_fingerprint":"289013f0aecd48cc3826e9cb944f60bf12cfa72939b6b4b1f2e235040d753b44","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_433aae1216ab2fb818f7bcdc","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":315,"at":"2025-11-24","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24","status":"stale"},"measurement_id":"aa22b5a2d1b6d59e300da9245c6d61426db6ed21252220b62a9126bc634aa5b2","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"standard_error_points":2.6971909527,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":"2025-11-24","source_scaffold":"Droid","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Droid","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"8b555c03eb29f74d5a3b90ab2125e8b06a3ec93e98b37a7323bf2869fa66b45c","result":{"confidence_high":68.432561682992,"confidence_low":57.859573148408,"sample_count":89,"score":63.1460674157,"score_display":"63.1","source_stated_rank":38},"run_fingerprint":"e596ad2ad7449617406ec6731cbd2fc8c1a68ce0f9ff621b880535bc85fb6892","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_433aae1216ab2fb818f7bcdc","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":315,"at":"2025-11-24","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24","status":"stale"},"measurement_id":"d26a359d02868942f3eed9982179baee4602bb7f856b73b098da004ded7407cf","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"standard_error_points":2.4124409274000005,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":"2025-11-24","source_scaffold":"Letta Code","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Letta Code","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"1aaa0022959e4e056edc3bfb844793c490d5c242384be067f6f6a89f84b3fa59","result":{"confidence_high":63.829507813204,"confidence_low":54.37273937779601,"sample_count":89,"score":59.101123595500006,"score_display":"59.1","source_stated_rank":51},"run_fingerprint":"3e8c8c7d52dbf4148af31e47ceb20f57874064fb825b26e51cf5aa67fe35032b","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_433aae1216ab2fb818f7bcdc","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":315,"at":"2025-11-24","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24","status":"stale"},"measurement_id":"242cae44263048ef37adfb970f218569c0cb28e6cab8fe2846eab59158d3aa99","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":"2025-11-24","source_scaffold":"Mux","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Mux","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"16ce50edb436be008079793d15e1813b3dfc094900d97acdef9801405c93ca64","result":{"confidence_high":null,"confidence_low":null,"sample_count":89,"score":58.426966292100005,"score_display":"58.4","source_stated_rank":53},"run_fingerprint":"20bdb123f3778fc83998f45be8e8e5cfaaa01dfc71ad58c518d71c4ec046e6ce","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_433aae1216ab2fb818f7bcdc","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":315,"at":"2025-11-24","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24","status":"stale"},"measurement_id":"37742507f826ef83abd92984bd8954f8ac5b5c82ab2c8e9045e35e77ad40ca6a","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"standard_error_points":2.5301893904,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":"2025-11-24","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"05139b330d9671b2c7d0c33603c6a945b15f17d0f14a9588319c9000d74671d9","result":{"confidence_high":62.711980193984,"confidence_low":52.793637783616,"sample_count":89,"score":57.7528089888,"score_display":"57.8","source_stated_rank":56},"run_fingerprint":"afe2ecea1cd91828063401ae5afb66df3d910d6aa9406500b8dd24de1c8da37e","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_433aae1216ab2fb818f7bcdc","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":315,"at":"2025-11-24","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24","status":"stale"},"measurement_id":"142d3dbc30b1acc2d95e256b2a158a45e4496aa9fc48313b0a8e33ca984d4913","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"standard_error_points":2.6265894906,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":"2025-11-24","source_scaffold":"Goose","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Goose","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"b9e8c6a300d06e27bc391787a54dc9dde6822ee271b7a947ada1e898c662406e","result":{"confidence_high":59.473958098176,"confidence_low":49.177727295024,"sample_count":89,"score":54.3258426966,"score_display":"54.3","source_stated_rank":62},"run_fingerprint":"4f384dfcbfe36801507427418e36988a6e2ecbeaa0c5869c53e2ee4f8a76e28c","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_433aae1216ab2fb818f7bcdc","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":315,"at":"2025-11-24","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24","status":"stale"},"measurement_id":"0b1c392e0f8b580fa03407d8339432a7e527997aec84393d15b00ac393e7ed93","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"standard_error_points":2.5301893904,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":"2025-11-24","source_scaffold":"Claude Code","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Claude Code","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"10798172477b6cff714631a8b64423ceb9185f4a8a860b769d8c1ec5b1177ff4","result":{"confidence_high":57.094002665884005,"confidence_low":47.17566025551601,"sample_count":89,"score":52.134831460700006,"score_display":"52.1","source_stated_rank":68},"run_fingerprint":"25dff8d14f7148ecaf4466c3db6505682d04866166f0321afc79e67757fa9d48","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_433aae1216ab2fb818f7bcdc","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":315,"at":"2025-11-24","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24","status":"stale"},"measurement_id":"bea4903ea448fc3c9e4aa6d288428aceef68a1bd7831c8af688806ebf1636115","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"standard_error_points":2.8882201141,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":"2025-11-24","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"2ee9779bd0c66ec4ecf0d4889afa457d03ace9205eb98a432047c395a7d1b384","result":{"confidence_high":57.571023783236,"confidence_low":46.249200935964,"sample_count":89,"score":51.9101123596,"score_display":"51.9","source_stated_rank":69},"run_fingerprint":"58991c4b20440b7773a23f32e46f403432dbb83f84794edd85449d0d49e9d536","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_433aae1216ab2fb818f7bcdc","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":315,"at":"2025-11-24","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-24","status":"stale"},"measurement_id":"5e55e7c4faf90c5facf5dc1c7b681c7265055a2af623cd2343ae4050a7c7144f","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":"2025-11-24","source_scaffold":"OpenCode","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenCode","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"c007a12a6d7b5975a2603f1a0f16be4c63825508b68ddccc97f1b75f430e909c","result":{"confidence_high":null,"confidence_low":null,"sample_count":89,"score":51.685393258400005,"score_display":"51.7","source_stated_rank":70},"run_fingerprint":"ac28c3be93e0e92601aff34f5fd78feb29e55da431bd1c3138dbba59cc665eec","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_433aae1216ab2fb818f7bcdc","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":153,"at":"2026-05-05","basis":"evaluation_at","evaluated_at":"2026-05-05","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"aecfa0162eaddb731046a73138a5f0d2622882b523467ca86f140c7f1ac0a50d","metric_details":{"cost":null,"pass_2":17.18,"pass_3":13.66,"pass_4":11.34},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"claude-opus-4-5_sierra_2026-02-26","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"67f26e9643b98a8dfb5466fbbc338423afd93b9175dbd49cdf6dc026e61077e4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.74,"score_display":"24.7","source_stated_rank":null},"run_fingerprint":"d35b7610077691877b4c2d164d4af63f33b91bbd60eb925f6e2356677f93a7f3","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_38e056a3f25f42f22b205781","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"4616f07f33815d7f8086c59606909999a8575034cfa43f6f2d22f534ada63135","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.371","mean_standard_error":"3.2","model_release_date":"2025-11-24","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.1,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"c8f8b6af0ca2c3137f1a769bbc67843a0fb040b9e596cb812a79d774b40ae9b5","result":{"confidence_high":26.776,"confidence_low":14.623999999999999,"sample_count":480,"score":20.7,"score_display":"20.7","source_stated_rank":35},"run_fingerprint":"85cd2215e009f91e07240c842bfb94da2baab0ff50f7d16e55e098375ad6b4d5","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_553c63e3252d94983ebd2bfa","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:03:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:03:18Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8d22944d7d361759f06cf27568ea3c19c9cebab1ef34ee5584a9f1ecb62ddf08","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":4.38,"leaderboard_last_updated_at":"2026-04-12","license":"Proprietary","model_link":"https://www.anthropic.com/news/claude-4","total_cost_usd":86.55},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"FC"},"run_count":null,"scaffold":"FC","tools_allowed":null},"protocol_fingerprint":"6a9c99bccbca11748bdf70a2bdd6eb864726c94bf4a029f84ab95898fe96d192","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.47,"score_display":"77.47","source_stated_rank":1},"run_fingerprint":"b16b927b0fc6a8c5c57ac2a62bb16f71d47b0ce143c2e4603aa9c1a7cfa7859a","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_add9f33c77552c619900f72b","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":38,"at":"2026-08-27T22:30:42Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:42Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5c8df24fd98df1ed05b8aee1f16b3cafa441b8dc3b1ec5bafd07b7f6907f62ae","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":3.76,"leaderboard_last_updated_at":"2026-04-12","license":"Proprietary","model_link":"https://www.anthropic.com/news/claude-4","total_cost_usd":88.33},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prompted-function-calling","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"Prompt"},"run_count":null,"scaffold":"Prompt","tools_allowed":null},"protocol_fingerprint":"564fbea687964bbbeb4c2a4d92f977e2c00894a357d9ecf0f2b928b7737d37ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.47,"score_display":"33.47","source_stated_rank":57},"run_fingerprint":"51d414c0adadefb7c07192c6a4c13c7f3cfe0eb8a161eeeb8d3b18764cc2b943","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_add9f33c77552c619900f72b","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prompted-function-calling","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6e3ab12c4773270b9df0879026673b23478cd1213bd7556afa94b386a4c3a35f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"d0d6155566171de54f7bb1d6fc1fbe71b75e8ef263810485607e592935427ef8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4967.056666666668,"score_display":"4967.06","source_stated_rank":30},"run_fingerprint":"b0040dcc613e9c1d3442cfb5cb557fecb5aa89133a94ec2f7bde54c6d00b3ba6","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acc9610846795e84cd094615","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval","metric":"win_rate","name":"GDPval","release_id":"release_730527fa346470703bd19e21","split_or_window":"expert-comparison-win-rate","unit":"percent","version":"GDPval official leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:05Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:05Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6dba6ebb7e82732825c2c6268c092926e180a24520745f3fc3f8b3cd70e9cb0c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-24","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-5","name":"Claude Opus 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.5","source_effort":null,"source_model_version":"claude-opus-4-5-20251101","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenAI GDPval expert-comparison protocol","tools_allowed":"as reported by the GDPval leaderboard"},"protocol_fingerprint":"378fc1a8e117ab4f314c4ef7da05eccc63a48ee1c23b7e95a30acb27f82c58b1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.5,"score_display":"45.5","source_stated_rank":2},"run_fingerprint":"f4dac89c9651116a6fff413eba7f186675348b339b5407e2a5196857bfcaf1af","source":{"content_hash":"d4eac25e0863d96ee45cc545fc896e5667d7022da8b3bc3902533ed87fa8b836","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gdpval","id":"epoch-gdpval","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GDPval","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gdpval"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b5b92874a5c47d9cc6e62f91","provider_config":{"source_id":"epoch-gdpval","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7da0118e5c62a1dde0fad55f2796ae019790f6473785e49a7093971faf850376","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-06-12"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":false,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Kimi K2.7 Code; model release: 2026-06-12","source_accessibility":"Open weights (unrestricted)","source_model_name":"Kimi K2.7 Code","source_model_release_date":"2026-06-12","source_model_versions":["kimi-k2.7-code"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4e334549e33dcace0decb9bb2f9d9d16b0177510964c7e0ea87330f8548874dd","result":{"confidence_high":151.75,"confidence_low":148.05,"sample_count":null,"score":150.01,"score_display":"150.01","source_stated_rank":50},"run_fingerprint":"65870276748eb86e9597914c3920f82dd4f745da6d8cee1da18d49f645c717c1","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_985db51e44c0b3d3bb9fbbfb","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"56618370543f09a1a51dba0ebe0026424dea0708aaa208d2219c561a537ce7a2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-12","source_locator":"Leaderboard models table · Kimi K2.7 Code · intelligenceIndex"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k2-7-code","upstream_model_label":"Kimi K2.7 Code"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"2ffca19e9d50d3835e933e45802adb4b271e85543507458587bd79b5310ff03e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.812106,"score_display":"25.8","source_stated_rank":null},"run_fingerprint":"11d1fb800b7b00e061c51194aaf9b82c23ac74bf13194d07f41f2011dad4ff4d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.7 Code · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c418354814777762766b7183","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"bf629d91200a4ef6c448dd1e84b5411109e8488f029983d06f5960a93c59b768","metric_details":{"category_scores":{"Agentic Coding":45.65633333333333,"Coding":73.959,"Data Analysis":62.65733333333333,"IF":56.29175,"Language":77.90566666666666,"Mathematics":79.60425000000001,"Reasoning":82.80775}},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.41172619047619,"score_display":"68.41","source_stated_rank":58},"run_fingerprint":"c50007a7beda3b74048b357cdac04f1808c6bbb4781d0efa2ddf9638dd50d560","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_47aaeb01147213e8437e7340","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2699aa5048a04a6558f496cc1046886a88b737e9de69e554cb07867135e17165","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-12","source_locator":"Leaderboard models table · Kimi K2.7 Code · hle"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k2-7-code","upstream_model_label":"Kimi K2.7 Code"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"654cbec8bc90e9b15b4ffadcac2dc4e1d59b86b4e9f72aaff382c966028a8d26","result":{"confidence_high":37.070282,"confidence_low":33.047787,"sample_count":2158,"score":35.032437,"score_display":"35.0","source_stated_rank":null},"run_fingerprint":"275afff318a875ca419a80d4e4d19439a856e7d57e5f4c10e814f39c4137c2ea","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.7 Code · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c418354814777762766b7183","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:36:03.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:36:03.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"17738ccaa95607a95ced32e3c153bbe3424f764f2b683d8307aa514d37c7eeb7","metric_details":{"best_score_across_scorers":"0.365","model_release_date":"2026-06-12","stderr":1.5231776226264846},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"5AijYw5bFJtSuahnbzNo6Z","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5AijYw5bFJtSuahnbzNo6Z.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/5AijYw5bFJtSuahnbzNo6Z.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"aa2afddb6862878a83c16956406d08720ef942d16c7bcb30f7d35c9d8326149a","result":{"confidence_high":39.485428140347906,"confidence_low":33.514571859652094,"sample_count":1000,"score":36.5,"score_display":"36.5","source_stated_rank":49},"run_fingerprint":"56bde1624438766e9a94a6ff59a75243ab67b6066af2f298979eb768d5f3b118","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_63aaba2756ec14ee77fae61c","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"64250df069d1be8d098dc68e53937b7c42de685ac841a5f4f46176d6c62e38dc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-12","source_locator":"Leaderboard models table · Kimi K2.7 Code · gpqa"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k2-7-code","upstream_model_label":"Kimi K2.7 Code"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a400dce6b0c306e359e607afc2aff88ff9d38261d3069ad2e05f70c85cc8b469","result":{"confidence_high":93.121301,"confidence_low":84.563379,"sample_count":198,"score":89.59596,"score_display":"89.6","source_stated_rank":null},"run_fingerprint":"d1e11db978baaaf93abd37db0b13ee741d87aa22d5e64e316b6d1183fc701333","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.7 Code · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c418354814777762766b7183","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:38:29.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:38:29.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d87cc262eff8f20b66c515c7c018088f50f04150f136be4e393b040077439616","metric_details":{"best_score_across_scorers":"0.8787878787878788","model_release_date":"2026-06-12","stderr":2.325315795194205},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"XneEig6pACkAWeekvgCwNk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXneEig6pACkAWeekvgCwNk.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/XneEig6pACkAWeekvgCwNk.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"a3b0eb031da0403cb63a887d16d9f8903f725be762a0cd6ad0e1f9999ee1baac","result":{"confidence_high":92.43640683736852,"confidence_low":83.32116892020723,"sample_count":null,"score":87.87878787878788,"score_display":"87.9","source_stated_rank":63},"run_fingerprint":"e87479f4ef8602d161ef6baf1c41be84dc71d97b139400c5b979e95d486492b2","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ea5c6f5540f071707dd044b0","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:20:43.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:20:43.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"65916eac4961675d3c2a10645e091509ea163288b4e6dd9f061a279547f55386","metric_details":{"best_score_across_scorers":"0.9555555555555556","model_release_date":"2026-06-12","stderr":3.1067790907534736},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ENc5ZM5TpBYNKkTWN4xyQM","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FENc5ZM5TpBYNKkTWN4xyQM.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ENc5ZM5TpBYNKkTWN4xyQM.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ba1560a0e78954e95a1fbeff6bb7dc8f519b825a9793b92f0c72e4c4343ebbcb","result":{"confidence_high":100.0,"confidence_low":89.46626853767874,"sample_count":45,"score":95.55555555555556,"score_display":"95.6","source_stated_rank":38},"run_fingerprint":"b453d31eb4f9ac62e6f21e7fd6cf843704c4bd4107969096fe528bdde60c384e","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a87929f64e446284534a7d50","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-13T01:39:44.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-13T01:39:44.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"41729887784d6226fe62de1a992bd14d1ae81559c7b789d8c2d31b66098fac7d","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5403508771929825","leaderboard_private_problem_count":285,"model_release_date":"2026-06-12","public_example_count":10,"stderr":2.957276813514758},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"X3RxRkt2pBUXYJjAQHsUo6","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FX3RxRkt2pBUXYJjAQHsUo6.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/X3RxRkt2pBUXYJjAQHsUo6.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Kimi K2.7 Code","thinking":null,"upstream_model_id":"kimi-k2.7-code","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"c934cab44e6163fc127623626f7cab37035e2979a6b4784688228eb71f9b820c","result":{"confidence_high":59.83135027378717,"confidence_low":48.23882516480932,"sample_count":285,"score":54.03508771929825,"score_display":"54.0","source_stated_rank":45},"run_fingerprint":"b1b98cde2d776da5eb5ed97aa3dc1c7072f4e5932a1327bd459446f5006f3e3e","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9137a8c8a7db7d39dbe479e9","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-13T01:39:44.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-13T01:39:44.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"90e034aa250beb0dbac873c5c37e76439996da5daba70c871b30ee53ba0f3257","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.12195121951219512","leaderboard_private_problem_count":41,"model_release_date":"2026-06-12","public_example_count":2,"stderr":5.173952057462543},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"UTVSgirxQpTgmTHbgE9EUk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUTVSgirxQpTgmTHbgE9EUk.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/UTVSgirxQpTgmTHbgE9EUk.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Kimi K2.7 Code","thinking":null,"upstream_model_id":"kimi-k2.7-code","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"cb655832235294bac51fdb680d75442eac73bd158681fbf20e452ca29f822269","result":{"confidence_high":22.336067983846096,"confidence_low":2.0541759185929287,"sample_count":41,"score":12.195121951219512,"score_display":"12.2","source_stated_rank":54},"run_fingerprint":"83e6cf6563f9c922dcf7d291cf1cd9cc408a774128cbdd0d3ed61d7108132981","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_37090ab2b47281e7d26ce1a7","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4841094f73376af94ca8ee92237b50c66a676a00ede8ddc4f920837ff78be3e2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-12","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.7 Code","source_effort":null,"source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"908f0d01d96bb008521601e6b0012cdb215cda4488d8d085b383ffdf10c74c0b","result":{"confidence_high":19.153356,"confidence_low":4.953024,"sample_count":71,"score":10.0,"score_display":"10.0","source_stated_rank":92},"run_fingerprint":"e2faa760580ad44531fdfc47e99ef5e030bf969865b3605a7edfc3ea0d92268d","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_63208020e242860d0a8be417","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e4da572760e1c2d5f5e948481de65be0c0c1f0dab7c18a15ab50c6e61a8b9ccc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-12","source_locator":"Leaderboard models table · Kimi K2.7 Code · critpt"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k2-7-code","upstream_model_label":"Kimi K2.7 Code"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"bdd9e15349dca29d0860072a913466deb663b2e4b1d21ee2aedf7fb5904f3307","result":{"confidence_high":19.233122,"confidence_low":4.928868,"sample_count":70,"score":10.0,"score_display":"10.0","source_stated_rank":null},"run_fingerprint":"f1c4ba6c0f8e2dfb5ab60294d52dfa55eaf89798f021cb3c9894d2585215f3f8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.7 Code · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c418354814777762766b7183","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1b829bd6d0f6a2d4f1dc44c80f269e57ae355617cb135c7888e5d78cafd08cc5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-12","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"bf92dc42aced9cbce2c8fb44c7e2bb56f53e75fb33c035f97e38139ed580cfd4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.9,"score_display":"57.9","source_stated_rank":37},"run_fingerprint":"cc4fc98d0ef8bca7943c67f7b53e0e4ee3c187e8b55eacf780aae8e15a968abd","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8861fafa0de5f1d143a949c9","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:38:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:38:54.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"64f9eb322424adfee55fccc39aa59d7a282444496ecb564ded18eb6fa4852c7b","metric_details":{"best_score_across_scorers":"0.21","model_release_date":"2026-06-12","stderr":4.093601807403323},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"k7EnhoBbZyVFUVRBowA5Pm","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fk7EnhoBbZyVFUVRBowA5Pm.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/k7EnhoBbZyVFUVRBowA5Pm.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0bfed3431bfe57f988dd1955090676c399ff6e323661be4b772e40493c08ac7b","result":{"confidence_high":29.023459542510516,"confidence_low":12.976540457489486,"sample_count":100,"score":21.0,"score_display":"21.0","source_stated_rank":80},"run_fingerprint":"bec962f59cfb54d308d919e9f9ed2681bec755c6d6661f960a2bba71724cc845","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a5fda2880c2096a50b322ade","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cb96caff8b6c5eced2f79d63629dd7158c44bb21cdb27ee85918821567326eee","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.265276,"date_is_proxy":true,"latency_seconds":896.71,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.848},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":null,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"kimi/kimi-k2.7-code"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"0904e69bb49c599abe2d467808d7a38767a6166d3db33dc21f23871a8639da13","result":{"confidence_high":81.82208,"confidence_low":74.57792,"sample_count":500,"score":78.2,"score_display":"78.2","source_stated_rank":30},"run_fingerprint":"aa6f288cb605c6a286d57ce5bf68ac7eebfe97c592e7be99b4614825bc076f7a","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4563edc5bc06fca8cdd968dd","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"848afee5ef88db978248d9efbcc83b4123d280b45f5665d0648c0c8e0fee7524","metric_details":{"ci_attempted":452,"ci_half_points":0.5007016522460913,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":138,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":149.1150442477876,"mean_cache_tokens":12857596.451327434,"mean_cost_usd":2.815536243274336,"mean_duration_seconds":2163.7950712809734,"mean_input_tokens":13008041.482300885,"mean_output_tokens":59297.24778761062,"median_agent_steps":139.0,"median_cost_usd":2.1955079100000003,"median_duration_seconds":1931.6495705,"median_input_tokens":10022682.0,"median_output_tokens":54054.5,"median_output_tokens_to_pass":48384.0,"median_peak_context_tokens":122628.5,"n_attempted":452,"n_passed":138,"n_tasks_attempted":113,"n_tasks_passed_any":69,"pass_at_4_points":61.06194690265486,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_kimi_k2_7_code_default","source_model_version":"kimi-k2-7-code","source_reasoning_effort":null,"source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"779874d29535932d55ae2983cd4c98afc7144b8b4301e0a615e94f526da87de2","result":{"confidence_high":31.031675103573527,"confidence_low":30.030271799081337,"sample_count":452,"score":30.53097345132743,"score_display":"30.5","source_stated_rank":63},"run_fingerprint":"c1e8cbed9a32399d6f611ef470ec52cffae1aadc3aeeff2ffd6eceee9ed501d3","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1d6cf902dce1551711bea365","provider_config":{"source_id":"deepswe","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_0b5df8aca801f09a2949c002","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE current leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T22:38:16Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:16Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"495a27d324e191946d6ff41ad1fe3157fc96402ca8def9892ef60a0b523c6dd4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_agent_steps":"149.1150442477876","mean_cost_usd":"2.815536243274336","mean_output_tokens":"59297.24778761062","model_release_date":"2026-06-12","notes":null,"pass_4":"0.6106194690265486","upstream_source_url":"https://deepswe.datacurve.ai/"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"kimi-k2-7-code","source_effort":"","source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":null},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"237f93c3fea21966effd66d3d727e205d33db763d36ff531f971031b06f89a94","result":{"confidence_high":31.031675103573523,"confidence_low":30.030271799081337,"sample_count":113,"score":30.53097345132743,"score_display":"30.5","source_stated_rank":62},"run_fingerprint":"99c5832f997ffbfdd5ed5d14a369a6cd054d3f679d275f35e6e4325c9960a13d","source":{"content_hash":"a418b0354d710bf9139250f1a2e7d0f64606aad80a7a24070d393ee82fa38aa7","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-10-05T12:33:23Z","homepage_url":"https://epoch.ai/benchmarks/deepswe","id":"epoch-deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"Epoch aggregate CC-BY-4.0; DeepSWE repository Apache-2.0","locator":null,"name":"Epoch AI mirror · DeepSWE","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/deepswe"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f6d0d706e38e4eef45a28e53","provider_config":{"source_id":"epoch-deepswe","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"bc285ce38013b2787bdf5ce9e3c1cdb04338a68a8f27f6e068fdd5426023c504","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.43","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-06-12","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"58","tokens_per_task":"31247","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.7 Code","source_effort":"","source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"06e6b9831545592defb2bc9160287490c92448b5f7b02e2afeb5f7a879b3bffa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.7,"score_display":"49.7","source_stated_rank":62},"run_fingerprint":"c4a131baacbdb58204649d7ef9298711066df3aa6c55d6b52f4c30aae2f81981","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0de78c1261d5008f214d3787","provider_config":{"source_id":"epoch-cursorbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1308247e750bafabcba38e0d047862ca2ca647040cbbb17d256ffb1f45c89e93","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-12","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.7","source_effort":"none","source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"227f91e8c423af202566cf13b45663433088c2b5a0344b93eabc4c5fd2951896","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":30.06,"score_display":"30.1","source_stated_rank":27},"run_fingerprint":"7dfe0fdf5a0fe4d46e9df5c54c24141efa0ef006304cba0dd723766d71c0bb8e","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_dd4b35536a13bf97a852f231","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1acd9136a9b0cafbbd6251f267de3d7c24d6b5be1f529c264bcf6c3e94f85df7","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.027349,"date_is_proxy":true,"latency_seconds":214.178,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.071},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":32768,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.7-code"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"2211f3762fbf251a33f60a313e7638e605a0af9dbb69c1ab59e7838c28234c23","result":{"confidence_high":84.14916,"confidence_low":79.95084,"sample_count":null,"score":82.05,"score_display":"82.0","source_stated_rank":58},"run_fingerprint":"7dc365b6c6544117022a5b57ee8284150ddc3e17a2d848c04caa0461d5b9aa05","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d975f32ded4665b2f1e01b66","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e434f6f935de2b553a31c615e9fda370a3690aaeedc95103715584f153ee5109","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-12","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.7 Code","source_effort":null,"source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e8e79e98bd24134c864da2b0e0dfbd0d74a2bdbc0d01c47a64397f02de275a25","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.453703703703695,"score_display":"47.5","source_stated_rank":106},"run_fingerprint":"d44016fbd2f82bbed2d41fa426cada22d82c5873914e7e19f477fcf134ae3688","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1c34c4174282cba541b7d9bc","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-06T00:23:09Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T00:23:09Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c9e864abc48014309afa84b052a7cb805b16525fada65d65d801cbafff8d122a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-12","source_locator":"Leaderboard models table · Kimi K2.7 Code · scicode"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k2-7-code","upstream_model_label":"Kimi K2.7 Code"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"c315f6b55a6e6f6f445d854e54e9c4af496a9593a1638aebfa11bfcccf4f0a70","result":{"confidence_high":53.560968,"confidence_low":42.098779,"sample_count":288,"score":47.800926,"score_display":"47.8","source_stated_rank":null},"run_fingerprint":"a8fd27e8e1f1b721e678fcea1e724f0ac507cd64394370a81c40d300f148bcf5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.7 Code · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c418354814777762766b7183","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8bd83734b1750b07c0d8ad66324702ee71c25d0d6e0b0b4e4539e405b3e7a14b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.184743","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-12","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"kimi-k2.7-code","source_effort":null,"source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"e08b8bef922d2365af23dc1a66a5964759321622b95ad38de5d62684f7c89410","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":886.23,"score_display":"886.23","source_stated_rank":61},"run_fingerprint":"ad9cc32243a714ab8537b3ef02ca2bc23eeaf182d8ffecf7a3b664f3b4a97e0e","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_623f2c3e7cb74eeabc3b0ea1","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f3e86229f777c5db5362b182b264a729b18be32495b438d8f27f64f09e9a51e7","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":3.835806,"date_is_proxy":true,"latency_seconds":10318.246,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.191},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":32768,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.7-code"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"df1de84f8b883fe86859ed6de10566e58b21ea2fa425935d895a46e567c1cda2","result":{"confidence_high":57.38536,"confidence_low":37.03664,"sample_count":null,"score":47.211,"score_display":"47.2","source_stated_rank":58},"run_fingerprint":"da8c3da31098d9d32d44411a650e3516fed06805dde2aed2543dd695e2cd63e5","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e042f5db1ee3a91dd7a3bc95","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2b3eaf3f5101e9a8c82bc916d7f7c6db4c480b71ffe164686c1170703ddca90a","metric_details":{"license":"Modified MIT","variance":22.882909725336816},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"kimi-k2.7-code","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"574658adcb883334260405d7f5897e588ec76baa6e2b05b318aae0213ee61ad0","result":{"confidence_high":1482.2764221444686,"confidence_low":1463.52502161997,"sample_count":4993,"score":1472.9007218822194,"score_display":"1473","source_stated_rank":61},"run_fingerprint":"03e00c2816b16d2ad536f41d3ffbc573e2a6dc484d93912efa4b3d842496d262","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d8d2349529d194d11d6a5ea0","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"87343d92d3082cfd40dbfcdae5cffcf298e37329cb293434a7378db8c8da4fb8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-12","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"4334c34036b06eda8649e07b05ed1288473fa3e3aa2ed3d6cf356986e11ec6c5","result":{"confidence_high":1482.38,"confidence_low":1463.06,"sample_count":4683,"score":1472.72,"score_display":"1472.72","source_stated_rank":60},"run_fingerprint":"1bda96a46253823934f219268e58f850a92587e1a2e16889ad36a62370563400","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5c787b6bad31d0c30c5a2b47","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1a50b8212240d4f8adb479936ab35e0b86194b2c0222095bd1b2e24b0455fe94","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-12","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"e3029412c8500dac68ebb1de4e59896852700e290bd965b86f9f8cc7d11f9fb9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.120000000000005,"score_display":"54.1","source_stated_rank":62},"run_fingerprint":"ef9357d32d48107f2d6191c05b99ee6cd09a7c9b5c744cae4e2c97aec133b27a","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85f25685b1561fc4451541c5","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3d7eb1f268982ed921be8fb7e8ceef8a9010a4eacb72f9ae6a82f46b1c6c447a","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.256503,"date_is_proxy":true,"latency_seconds":784.159,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.375},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":32768,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.7-code"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"bac6b5e661d6a47f7a5e62ad28ee9c9685fc81e2199436e7de610c9c43bea81c","result":{"confidence_high":67.776,"confidence_low":66.306,"sample_count":null,"score":67.041,"score_display":"67.0","source_stated_rank":36},"run_fingerprint":"61c371b9b3e2b9ed6767b77fef682117b0ffbcbad9e1b4e1fb80159aa0b29a45","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f7cb373722491bb55787c0f1","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aa3bc17596cb6aa1736061bdc85f747a42a21e9ea30d467aca78319a4cd96e6d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-12","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Kimi K2.7 Code · terminalbenchV21"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"kimi-k2-7-code","upstream_model_label":"Kimi K2.7 Code"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"0f05a9f431ea5c7cbbe97d7c32ccba3318502738a4ba9302b4cacdec26002926","result":{"confidence_high":76.256171,"confidence_low":57.134033,"sample_count":89,"score":67.41573,"score_display":"67.4","source_stated_rank":null},"run_fingerprint":"c58068463656fe90711704dbd908c3161d2dd7f8f79aad60fbfadb14169ed4db","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.7 Code · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c418354814777762766b7183","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5b84e6c617353fcf4b55dd3a612e7b9ceec527110fa85894c0fb30b96c81500f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-12","source_locator":"Leaderboard models table · Kimi K2.7 Code · tauBanking"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"kimi-k2-7-code","upstream_model_label":"Kimi K2.7 Code"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"c9e2e626ec237996b0057e33b76837c1c7592088e24b3d167d9861d2df8befdc","result":{"confidence_high":29.260191,"confidence_low":13.422195,"sample_count":97,"score":20.206186,"score_display":"20.2","source_stated_rank":null},"run_fingerprint":"284bbf3d78d562eab3f45af23305230640e3e382bae102eb7e9576865f6e5a72","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.7 Code · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c418354814777762766b7183","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aa6168d79168bf6415c499236fdc721d828108bf8d4f1fef65ca40c8f4f0468f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.534","mean_standard_error":"4.4","model_release_date":"2026-06-12","notes":null,"standard_error_points":480.0,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.7 Code","source_effort":null,"source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"dd46b5d88e1af0751ee332c71ae1a197bb86daf935107d3c43b626d7dd376f65","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":37.6,"score_display":"37.6","source_stated_rank":34},"run_fingerprint":"e2172ca00a3b502cc4110f1e35048b1472015d16ef38b7281516afd9cf09c890","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_099c0ba8b21819b771db4fc6","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"08c6d87b4a268ae5e04854387997bc89f97e3d3e0742ea93a2e80094ac8a90ee","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-12","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.7 Code","source_effort":null,"source_model_version":"kimi-k2.7-code","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"38be7dd9ac77751164079289b317909d6a2c57075ba33d292d0899260c6fefb5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5082.936,"score_display":"5082.94","source_stated_rank":28},"run_fingerprint":"4298ce0934ebe6239e5087158fb075d407714cf22f233c9ec89b84b2a3ecd668","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b04515110e6907c541185f5","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b7018fdb17c9ded43c19439ed2cf438ee9a8bf0668959d485c4767b8531e1e91","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-12","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Kimi K2.7 Code · gdpval"},"model":{"id":"moonshot/kimi-k2-7-code","name":"Kimi K2.7 Code","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"kimi-k2-7-code","upstream_model_label":"Kimi K2.7 Code"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"11e0834f4d8fbfc4861381dc998210eebadeadd678b6990fd9a2779de28c20e3","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1025.04,"score_display":"1025","source_stated_rank":null},"run_fingerprint":"8647e17f3a4987b5cbad44270336869dc551f4c59676fb39aa28dd2f7e70ecad","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.7 Code · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c418354814777762766b7183","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bc787a738b0224499290fa77bffb1dbdc757a0cd352a835ef47d1e298dc09d0f","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-08-07"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5; model release: 2025-08-07","source_accessibility":"API access","source_model_name":"GPT-5","source_model_release_date":"2025-08-07","source_model_versions":["gpt-5-2025-08-07_high","gpt-5-2025-08-07_low","gpt-5-2025-08-07_medium","gpt-5-2025-08-07_minimal","gpt-5-2025-08-07_unknown"],"source_reasoning_efforts":["minimal","low","medium","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3d0d71ae1fc5c2107583eb7ca8cdfc64da0015b7865cb9be57c5e246f15c5f96","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":150.0,"score_display":"150.00","source_stated_rank":51},"run_fingerprint":"a8feff62ade75267e78292c263ff7bf67cd2529b0d93b98504b07ac07a4c93df","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1cb8f18b8ce4151abd957ac3","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c11ef60fed9ef5d2834321c42fe81b4d687d0fde3bff07bcdebae36d1088aacf","metric_details":{"license":"Proprietary","variance":5.319373491862773},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1439.02384421239,"confidence_low":1429.9830136707012,"sample_count":31489,"score":1434.5034289415457,"score_display":"1435","source_stated_rank":105},"run_fingerprint":"8ad1f3d443fb8ccc663eeacceadedcecae1e01c0f2bb9618c6135d431b036d54","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:28:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:28:49.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"28658bf55fb07563cf19c052c223f04aa45ad1b516fe2301df7e80b75e4a5688","metric_details":{"best_score_across_scorers":"0.501","model_release_date":"2025-08-07","stderr":1.5819268290576818},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gTLurbyKU63i2opsGkJGwh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgTLurbyKU63i2opsGkJGwh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/gTLurbyKU63i2opsGkJGwh.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d189d8329e6b49553a7c46940840debb36916abe035715f2efcbc90e351d5545","result":{"confidence_high":53.200576584953055,"confidence_low":46.99942341504695,"sample_count":1000,"score":50.1,"score_display":"50.1","source_stated_rank":25},"run_fingerprint":"d391af9cfad5c03131d3e664a904ad07d5aa6a1a218ba32a799f3598cdea4f60","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"adf80b7d7990c4e6c6e903f03cdd6d7341a5d2591b0530ebb7764270206112d1","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.072708,"date_is_proxy":true,"latency_seconds":169.716,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.764},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":84.848,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5-2025-08-07","zero_shot_accuracy":86.364},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"35f1f5a6877d9f44fe2ebe48e2b1f7b6b9aeabf0beced3e72d9e8d49f0d62360","result":{"confidence_high":89.06344,"confidence_low":82.14855999999999,"sample_count":396,"score":85.606,"score_display":"85.61","source_stated_rank":46},"run_fingerprint":"ecf1fb0183059d2c8ffe39fa2b0711e4cca1cb4e3a5c6a1b579476c77182b6c9","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":340,"at":"2025-10-29T22:29:36.692Z","basis":"evaluation_started_at","evaluated_at":"2025-10-29T22:29:36.692Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"df424c925b3d934ad231233d3a06b8621c54b419427401c40df407a9466a3926","metric_details":{"best_score_across_scorers":"0.8617424242424242","model_release_date":"2025-08-07","stderr":2.0787836099851997},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"9nPyNrZxwtoT7eS4DZbQF6","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9nPyNrZxwtoT7eS4DZbQF6.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/9nPyNrZxwtoT7eS4DZbQF6.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6f12e35d7a079a9e22f0202a61b03afd05a118b1286fd15bb45e0a5ae330535f","result":{"confidence_high":90.24865829981341,"confidence_low":82.09982654867143,"sample_count":null,"score":86.17424242424242,"score_display":"86.2","source_stated_rank":80},"run_fingerprint":"962bd03fff2d4b5ca7b5ce521308a8e8925bf7b6c354138a424ce98a452429f7","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":76,"at":"2026-07-20T18:29:01.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-20T18:29:01.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0768b7e28b013873485eeb1209199af2a345b5390af7f6d4e089d1e69ffe5603","metric_details":{"best_score_across_scorers":"0.7171717171717171","model_release_date":"2025-08-07","stderr":3.2087795587867514},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"n4wyt7wHjhDdjrGacE9smk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fn4wyt7wHjhDdjrGacE9smk.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/n4wyt7wHjhDdjrGacE9smk.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"5d0f4783fac2917e3a78f6927ff1b3b492556044012b77963617518b61735677","result":{"confidence_high":78.00637965239375,"confidence_low":65.42796378194967,"sample_count":null,"score":71.71717171717171,"score_display":"71.7","source_stated_rank":173},"run_fingerprint":"f43a6fba2a8e0c68e0e3e51ec2c0d30cfd958979adc059e6fc3076b6b28f3a4a","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"28806b84ce4ee902c3b8c66efbcf3fd1f31913315fe5f75997998209fda5e90e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.024135,"date_is_proxy":true,"latency_seconds":37.239,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.336},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5-2025-08-07"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"cfbea1127455a41e3a0bb415b106fd5fc660fc3d92df797cd11cf38acbecb600","result":{"confidence_high":87.20255999999999,"confidence_low":85.88544,"sample_count":null,"score":86.544,"score_display":"86.5","source_stated_rank":41},"run_fingerprint":"8a9b37b79a97f1ca1527435291c47c50579a8c67cde7195ea39e49aa1c5b7fc4","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":340,"at":"2025-10-29T22:29:41.753Z","basis":"evaluation_started_at","evaluated_at":"2025-10-29T22:29:41.753Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f91dc03f60962861ee41dfbcfa6cf7023151b8f6b0d6a5ffb57dfaf21fd5adb1","metric_details":{"best_score_across_scorers":"0.9138888888888889","model_release_date":"2025-08-07","stderr":3.658872186755414},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"3D8aaFBLZwQcnWHYKwvnmq","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3D8aaFBLZwQcnWHYKwvnmq.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/3D8aaFBLZwQcnWHYKwvnmq.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c186d45c9a9fff2bb52e86b3267dedec98985f992f90269037226d406788d15c","result":{"confidence_high":98.5602783749295,"confidence_low":84.21749940284828,"sample_count":45,"score":91.38888888888889,"score_display":"91.4","source_stated_rank":63},"run_fingerprint":"d79497ffc2379e4a5197460d7223194e2f41a4973c76b9bdf783a21571f07926","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":76,"at":"2026-07-20T18:28:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-20T18:28:50.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"63d03f0b309bd6b629b6d1f516e00f5f89622b81786599a41822132ee267f16c","metric_details":{"best_score_across_scorers":"0.4666666666666667","model_release_date":"2025-08-07","stderr":7.521014330903548},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"RMsr7t8ZwJmox4m863sJGm","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRMsr7t8ZwJmox4m863sJGm.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/RMsr7t8ZwJmox4m863sJGm.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5f39f2a8b2a1a2318878c03749c397e7ed9377d9997c1671a6c6320702fe5897","result":{"confidence_high":61.40785475523762,"confidence_low":31.92547857809571,"sample_count":45,"score":46.666666666666664,"score_display":"46.7","source_stated_rank":193},"run_fingerprint":"4eb9f74ebc0074c4a01c7cf3318a2d259633aab3539835dfab148f901c6b8aee","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-10T20:10:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-10T20:10:57.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"290c08550f2b0cc445c84557cf517e393062b4fc27ff22a0f2366fa995643034","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5543859649122806","leaderboard_private_problem_count":285,"model_release_date":"2025-08-07","public_example_count":10,"stderr":2.9493504108183504},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"QRDHTGX3rYrVCjVPRarjjt","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5 (high)","thinking":null,"upstream_model_id":"gpt-5-2025-08-07_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"24bfd57e52b17e93343b428d62a99f1a061793a8d5eff9b7ea76cd79568884e6","result":{"confidence_high":61.21932329643204,"confidence_low":49.6578696860241,"sample_count":285,"score":55.43859649122807,"score_display":"55.4","source_stated_rank":43},"run_fingerprint":"d0ae787d0fa2d0f2e769b75500064f06a67a79c0ab2d4fb650800f46c23e8067","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:59:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:59:52.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"613319e14beb3653de7ac15544ef42571f84f0ec198f0d456d1fec3d040230ce","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.3719298245614035","leaderboard_private_problem_count":285,"model_release_date":"2025-08-07","public_example_count":10,"stderr":2.8679753751074615},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["low"],"epoch_id":"J2DWDpGLBhihZx82M3hhug","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJ2DWDpGLBhihZx82M3hhug.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/J2DWDpGLBhihZx82M3hhug.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5 (low)","thinking":null,"upstream_model_id":"gpt-5-2025-08-07_low","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"c3234427e4dc0a20fcd00b6ff9bb362e54b3e035eb3a1222353884cdc882949c","result":{"confidence_high":42.81421419135097,"confidence_low":31.571750720929725,"sample_count":285,"score":37.19298245614035,"score_display":"37.2","source_stated_rank":59},"run_fingerprint":"3cb272cd405a353462d3f545678130e2bb6031eb3700ac62910c04986323c75a","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0022c6c4b1de1d1aa7fa0672","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T17:53:33.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T17:53:33.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7998c936d5de491e66a54cbaf0719a583219380b2f3b3928778e0900a6794ce7","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.1824561403508772","leaderboard_private_problem_count":285,"model_release_date":"2025-08-07","public_example_count":10,"stderr":2.2917941277334046},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["minimal"],"epoch_id":"53PtW8g64r6zeLY38aK9Q4","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F53PtW8g64r6zeLY38aK9Q4.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/53PtW8g64r6zeLY38aK9Q4.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5 (minimal)","thinking":null,"upstream_model_id":"gpt-5-2025-08-07_minimal","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"893edba087c9f7142c84dc2dbc58157dbc0e322e889dc451a53282b67a6802ba","result":{"confidence_high":22.73753052544519,"confidence_low":13.753697544730247,"sample_count":285,"score":18.24561403508772,"score_display":"18.2","source_stated_rank":91},"run_fingerprint":"87dd526e1114501b2a1a9af2aa1f2ecbe954a8f61c3c09768dc155cffa74b35e","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fdc5d34198e5823e888ac97ab68cebacd1757c130de9ecefc00370a0698b634e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.7302,"model_release_date":"2025-08-07T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-2025-08-07-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"890afcba234e215d40e046278e01dede4804e61887e34e520bbc86d3d63ce9c2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":9.86,"score_display":"10","source_stated_rank":144},"run_fingerprint":"e2e2124e6d20440d32a188081ffa393402b5ae875e3256321db9d6af7a12e86f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"24acb444abf01154eaf94c63e217fc3137e8c381270f2e3e92478c5b4f9c8138","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.4486,"model_release_date":"2025-08-07T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-2025-08-07-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"70a85549cc311f075bed9c79c3cf5e35180f7bf30c67381fc4a0c07521a44047","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":7.489999999999999,"score_display":"7","source_stated_rank":149},"run_fingerprint":"c4c233ee21648d3426bd5a501059851c13d35ab5cee5985424a908ae742cca75","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"877f6706bb2730509a38327947c866d547550692d5fc32e2e642f2854dfd88be","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.1896,"model_release_date":"2025-08-07T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-2025-08-07-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"229ae402f69e48a3288f7a2b3314b20ee13c0011158e39ece7fb59c8c13db7f4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.94,"score_display":"2","source_stated_rank":199},"run_fingerprint":"968193f973700fd32bc341ff9b8722f124f295d359a1d9cdfe56c9c897cbca4f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0022c6c4b1de1d1aa7fa0672","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"49daed85cc0af6baf3697b86138cc2a230b53155c99bbcd66e4b1b5daa87e55c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0562,"model_release_date":"2025-08-07T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"","model_type":"Base LLM","upstream_model_id":"gpt-5-2025-08-07-minimal"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9b29ecbc59ab59d1c5c63f0086248f2bbcc402881c57070c0281fdc39771e031","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.0,"score_display":"0","source_stated_rank":239},"run_fingerprint":"d176ef1120c7da6a6927fa409dabac2746929028943bfd1e24433bd8c5ea031e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"337b9d253162b1eaafe295f849172eb3b1aba1343696e036e546038d10ff5869","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.7302","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (High)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"558bee05be2068d694ed0def7cab26885a3ca8d4f5fc86a2ff8df2cb049dd956","result":{"confidence_high":13.376348,"confidence_low":7.191284,"sample_count":360,"score":9.86,"score_display":"9.9","source_stated_rank":140},"run_fingerprint":"0db75567fe5f4010278aac7da6f972e7f1b97f6f5d2e5f567ca2d22b80a733f6","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a726ea6f03e1d0b583c77b06f4964b03f15f034743220420c1534dc2621b988a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.4486","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (Medium)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"650760f5c4dd7d28a7a076ed6f8a8d42c60d0423faf62a88e99e3c9790f0323a","result":{"confidence_high":10.680627,"confidence_low":5.197052,"sample_count":360,"score":7.489999999999999,"score_display":"7.5","source_stated_rank":145},"run_fingerprint":"14aa160c933c7b5905571edd5a8922602b9a697a116f4c8de6f44ce96d3289ef","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9223983946bf7f6e8fb213ae32edab1f11c81a2195517ff75dcc265e7fa9887f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.1896","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (Low)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d2a02a36884def15b64a5f290622a1c022639eb9c498cb21ff98435a6c5352d0","result":{"confidence_high":3.952794,"confidence_low":0.942084,"sample_count":360,"score":1.94,"score_display":"1.9","source_stated_rank":193},"run_fingerprint":"dbcf5386f0e3f9d354dd89298e801b3e4bf8519e90e9e30bd531923374dc96f3","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0022c6c4b1de1d1aa7fa0672","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0dd9247567c07834539be97bf4a6c30efac69fde491caea817cf6d19221ddfa8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.0562","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (Minimal)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"9766b54092ef41b8b9efafdc6dc1bea90f8feb582485c131c5673905e3cf2865","result":{"confidence_high":1.055844,"confidence_low":0.0,"sample_count":360,"score":0.0,"score_display":"0.0","source_stated_rank":230},"run_fingerprint":"883f2b08f53d4b7a7f1e0894d96d2bb71675f770c7e156514147afdb4763db56","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":116,"at":"2026-06-11T01:44:04.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T01:44:04.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"58c032c19fb0c4664148a6ad22efb3da4823e8b2de6b23b97a9085562bed078c","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.21951219512195122","leaderboard_private_problem_count":41,"model_release_date":"2025-08-07","public_example_count":2,"stderr":6.5445892024384085},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"mSS7C84hJyYi4VnJtgzYvW","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5 (high)","thinking":null,"upstream_model_id":"gpt-5-2025-08-07_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"ad242822e7c613453673ec02fd05b87f3c04a577fab7a7d24531fd85e26ac8e1","result":{"confidence_high":34.7786143489744,"confidence_low":9.123824675415843,"sample_count":41,"score":21.951219512195124,"score_display":"22.0","source_stated_rank":44},"run_fingerprint":"a340560b501e03f7ca0551594a2bbd56d174fbe1f313cae6a26b7d80bcbe0962","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"441855d9f9312ddfe8aad3740172174a26ffd726b546b569e329a50be4836d11","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.032532,"date_is_proxy":true,"latency_seconds":72.114,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.934},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5-2025-08-07"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1f24f3f89f19727b6d974638743e5b5518824b9bee9d99a0b13a22aad3d87c05","result":{"confidence_high":83.33364,"confidence_low":79.67236,"sample_count":null,"score":81.503,"score_display":"81.5","source_stated_rank":39},"run_fingerprint":"d172b93bf46af5feffa10e55404b1d818e438f8b1a818b9027f556af25c9f21a","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8cf52738895c36292967cca9960ddd30c8301bee47529bc030d665619d624a31","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","cost":"202.3","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (high, code & web)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"98765254d0059f0636723deec0d983f904f98ca5b1b04a5db94eb9475440e333","result":{"confidence_high":22.279355,"confidence_low":6.76011,"sample_count":71,"score":12.6,"score_display":"12.6","source_stated_rank":85},"run_fingerprint":"7b8f4650d374ab35d76adf5dec82d22aebfd075bbfdc0e73dd84df96570602cb","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9eb6d585085edecb8ef1297241ea5e300d1fab7ef50144ee4c8caf4a97166bed","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","cost":"8.4","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (minimal)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"39f727aaa7f895843130b708e3755e1503fbecbe654bb11c8a4296e1d5380caf","result":{"confidence_high":5.132974,"confidence_low":0.0,"sample_count":71,"score":0.0,"score_display":"0.0","source_stated_rank":167},"run_fingerprint":"b50427244424f1044f109a8322f37c407f87cb2cc8d82590fcd1799f521a5d90","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ac285bc139575e01db572434a4777b7eb2f254c1ea10d460a416675e5b3ff5c9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5-2025-08-07_high","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"d4b60efde0eed35629e96d2b3292df1b567f324a27291a7f98d280d05b39ccd3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.7,"score_display":"56.7","source_stated_rank":39},"run_fingerprint":"79da92c6de542273545816264d65d2a6c1a538279bae6dd5a6e074a6e14980ae","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":300,"at":"2025-12-08T14:58:41.672Z","basis":"evaluation_started_at","evaluated_at":"2025-12-08T14:58:41.672Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"08914842254e4a2afeb6596d639f6c80d45526d5c8fac39816448fb6476bca22","metric_details":{"best_score_across_scorers":"0.37","model_release_date":"2025-08-07","stderr":4.8523658709390975},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6DcmBdRZz57U5cusZNBeGW","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6DcmBdRZz57U5cusZNBeGW.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/6DcmBdRZz57U5cusZNBeGW.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6702531d9a02f97b193c6fd4b11aa3ecef3787932c1e73658bbd19551dc858e3","result":{"confidence_high":46.51063710704063,"confidence_low":27.48936289295937,"sample_count":100,"score":37.0,"score_display":"37.0","source_stated_rank":38},"run_fingerprint":"78bb7d5647952ef546502b18316ab6e3e5fee83d1fe00c2b75205480fc930016","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:30.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"433e301ba3be92596cbd9189d792a6057f37b9deb3ff841119a41b313b4e8f03","metric_details":{"best_score_across_scorers":"0.29","model_release_date":"2025-08-07","stderr":4.560480215720686},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ddktY6CXaXX9ZDCxFpRqyf","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FddktY6CXaXX9ZDCxFpRqyf.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ddktY6CXaXX9ZDCxFpRqyf.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"327319350f76996ffaf919f537179dfd3a20f9ec2414f64e45717864fb163ee6","result":{"confidence_high":37.93854122281254,"confidence_low":20.061458777187454,"sample_count":100,"score":28.999999999999996,"score_display":"29.0","source_stated_rank":54},"run_fingerprint":"94d00f63b053e695143733572313ed59b6d289cc36ba8f490e57a72969c162c6","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:27.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:27.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"944163d82bdbaa43eaca9a3be0d77067ae9cb9b195dc79ca28ded3f1d7e1ce22","metric_details":{"best_score_across_scorers":"0.24","model_release_date":"2025-08-07","stderr":4.2923469599092785},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"XDukYaTjaX5jCg4dxDkYjv","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXDukYaTjaX5jCg4dxDkYjv.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/XDukYaTjaX5jCg4dxDkYjv.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4b43f36b886599cf83d932d913b9158bdcfad5a54fdad9370ed0d0db0d0a1b9d","result":{"confidence_high":32.41300004142219,"confidence_low":15.586999958577815,"sample_count":100,"score":24.0,"score_display":"24.0","source_stated_rank":69},"run_fingerprint":"83a29ad686677d1f35f128f608af1a57498570f4e1f1b5c2aaab67feb4160b99","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0022c6c4b1de1d1aa7fa0672","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":82,"at":"2026-07-15T03:14:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-15T03:14:42.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cef9bf1a8347b188303f6bcc0aa3229d421290b89eab2afe626eb50ca14b3530","metric_details":{"best_score_across_scorers":"0.16","model_release_date":"2025-08-07","stderr":3.6845294917747067},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"AMXHrC3pFCznAFiReoTKBt","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAMXHrC3pFCznAFiReoTKBt.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/AMXHrC3pFCznAFiReoTKBt.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"305722ec88866f0c2c405caff4f1e5eee37374718bf6c5ed93beb3e2c0dcf896","result":{"confidence_high":23.221677803878425,"confidence_low":8.778322196121575,"sample_count":100,"score":16.0,"score_display":"16.0","source_stated_rank":109},"run_fingerprint":"834bd8559bef6396b61c66daabcab5dea2826d6f8ea7f5ecccdb9049cfd9bebb","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b2fb8def2fd92c581eb8f0d0e86629924ae3f2e4384b0ecbe004fa7fc2ebe8bb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.5087","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (High)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"76e873d45f3bda83d48349f4e829217cadcfd916987952eaceea32bbe82a4219","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.67,"score_display":"65.7","source_stated_rank":140},"run_fingerprint":"752e24090111ea24b013816ae7a5cdb7c4be864f1eb56b03cf26324f9f00fcb0","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cfd0ea4437ef11154a6f62195520c3afcf5118d640736895907de3f0e2599494","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.3301","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (Medium)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d0d612a915a78ec23e66ffa041149decdeed40133152de1fea48679e7b408021","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":56.169999999999995,"score_display":"56.2","source_stated_rank":160},"run_fingerprint":"a5eb4d302ed1c0424627f3da0266d43333df5d63f1823448dce728bc8b159219","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b3e85708a0056d250ac907810c7697b0656fdf61fe6d2c362e80c74cb9f9a4c0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.1531","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (Low)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"bb6ae1c1149f915def09462a71907ec3cbabff563efa2f77af3163318866f392","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.0,"score_display":"44.0","source_stated_rank":176},"run_fingerprint":"37995ceaa529162b8ca9780d16cebae7f0a660348dead03069c95f705228b754","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_0022c6c4b1de1d1aa7fa0672","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"564d46957ad3a59e9a07bfb962885f4a3f5e4d804072a15c9df2a41f6b854a92","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.0335","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (Minimal)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"53ae94551c0c7d4b04c3cb9ba112a6226985c94821cc1fd9431c20ba43dfb28e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":6.0,"score_display":"6.0","source_stated_rank":254},"run_fingerprint":"cea645181ccf55ad123fcc5e8c326a1cd6a0367c6513ab04567aad7d19941dbb","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"03701b53c03758258d1f874cb34b6fcb0db97000b86402fa28736015a38384c7","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.27029,"date_is_proxy":true,"latency_seconds":232.842,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.07},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5-2025-08-07"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"a45a668e439df6db4750e4f58c130e0acc13b08c7a2b7e570f63d34f7df21ef7","result":{"confidence_high":73.0572,"confidence_low":64.9428,"sample_count":500,"score":69.0,"score_display":"69.0","source_stated_rank":67},"run_fingerprint":"e2ff792c717d027bdb23711d923d2ddac9f14a7413567d52a83d2dc5b2ef4cc5","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":241,"at":"2026-02-06T14:23:51.990Z","basis":"evaluation_started_at","evaluated_at":"2026-02-06T14:23:51.990Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"7af0e93a80652b8acfb467948d3ee6f4de65b206345c21017a37ab6ca29cf6f4","metric_details":{"best_score_across_scorers":"0.7355371900826446","model_release_date":"2025-08-07","stderr":2.0068321817102888},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"C85Bnci9xVzq5jmt8zNnbw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FC85Bnci9xVzq5jmt8zNnbw.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/C85Bnci9xVzq5jmt8zNnbw.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"52a4a6405d4b4c272b321d4722ad797b51f74d75aaa5750c98e41615d5ed1345","result":{"confidence_high":77.48711008441663,"confidence_low":69.6203279321123,"sample_count":484,"score":73.55371900826447,"score_display":"73.6","source_stated_rank":20},"run_fingerprint":"0e51a5f1e4325d599318235a3c987da59ee57d12e58a3d0de19e64c3c9203cb4","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":242,"at":"2026-02-05T13:39:28.137Z","basis":"evaluation_started_at","evaluated_at":"2026-02-05T13:39:28.137Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"99cbd58fb92d32beeb55fe194cba749f914f4589384c8e15e51d0c2ee563355c","metric_details":{"best_score_across_scorers":"0.7148760330578512","model_release_date":"2025-08-07","stderr":2.0542754003902384},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FCCkMNVHcjLLUDeNFqTGRL","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFCCkMNVHcjLLUDeNFqTGRL.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/FCCkMNVHcjLLUDeNFqTGRL.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"04c54fe37de3e79913a0c75f630c6c3ed7151fc81ad7d630913da4781b8c0c5f","result":{"confidence_high":75.51398309054998,"confidence_low":67.46122352102024,"sample_count":484,"score":71.48760330578511,"score_display":"71.5","source_stated_rank":24},"run_fingerprint":"2c425c7bd68833f8953c98b157065c1a4cc054eff40a131a260914a0c0624b76","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":355,"at":"2025-10-15","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-10-15","status":"stale"},"measurement_id":"b9a0dd685b2f145d37ddc72cc7a5ed18315e42e902f5104c3d1494a06c571707","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"Prometheus-v1.2.1","checked":false,"date":"2025-10-15","folder":"20251015_Prometheus_v1.2.1_gpt5","logs":"s3://swe-bench-submissions/verified/20251015_Prometheus_v1.2.1_gpt5/logs","model_display":"GPT-5","model_org":"OpenAI","reasoning_effort":null,"resolved":74.4,"site":"https://euni.ai/","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: gpt-5-2025-08-07","Org: EuniAI","Org: Delysium","System: Attempts - 2+"],"trajs":"s3://swe-bench-submissions/verified/20251015_Prometheus_v1.2.1_gpt5/trajs"},"run_count":null,"scaffold":"Prometheus-v1.2.1","tools_allowed":null},"protocol_fingerprint":"6a931eceffdafc33c509cb58a21cccb332afa4bf6c3157887790ccfae6f0b70f","result":{"confidence_high":78.029292,"confidence_low":70.398626,"sample_count":500,"score":74.4,"score_display":"74.4","source_stated_rank":13},"run_fingerprint":"8f84b3881bcfd349c6a0f12b8921ceafaba9f9889c125e8fd0273937335e0547","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ad386cf9d2cf478c204d81d4","provider_config":{"source_id":"official-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b6135d4900eab3551994a8e2","split_or_window":"2025-08-15/2025-12-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":308,"at":"2025-12-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0e93086932f5028c78595102af577745c400a518d2bad50e86bb5f10aa54a077","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-08-07","pass_at_5":55.27950310559007,"potential_contamination":false,"sem":0.8002545792996965},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"gpt-5-2025-08-07-high__tools","window_from":"2025-08-15","window_status":"historical-post-release","window_to":"2025-12-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"11f968eae6635f7046fd3e21a385718c61f875fef2348c7dc08a49bc6b6c6e14","result":{"confidence_high":46.53744307480628,"confidence_low":43.40044512395148,"sample_count":null,"score":44.96894409937888,"score_display":"44.97","source_stated_rank":null},"run_fingerprint":"b879e2cb2b6a44b6ebaa9b28587d35f6349fc54a52ca33cc0b0518b8873bf68a","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b6135d4900eab3551994a8e2","split_or_window":"2025-08-15/2025-12-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":308,"at":"2025-12-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b0a57f59f0d1c5b00a3e321dcfbdc35409b163f23fbb0781499ba3db12621487","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-08-07","pass_at_5":56.52173913043478,"potential_contamination":false,"sem":0.7453416149068326},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"gpt-5-2025-08-07-medium__tools","window_from":"2025-08-15","window_status":"historical-post-release","window_to":"2025-12-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"0cde83ef8d3695dddb349077f2315f5eeae331c4d74cff1329b2b21a59ce5ef8","result":{"confidence_high":48.16894409937888,"confidence_low":45.2472049689441,"sample_count":null,"score":46.70807453416149,"score_display":"46.71","source_stated_rank":null},"run_fingerprint":"4144a9e035dd66c0f1c0f23e11f821bcafe2397166cc8015abf5eab71ec7ac02","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b6135d4900eab3551994a8e2","split_or_window":"2025-08-15/2025-12-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":308,"at":"2025-12-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8778c46cdad1a5edcb33d365fa658707cd34326ce13790204bd59e3bef7080d7","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-08-07","pass_at_5":45.962732919254655,"potential_contamination":false,"sem":0.9780008034476805},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"gpt-5-2025-08-07-minimal__tools","window_from":"2025-08-15","window_status":"historical-post-release","window_to":"2025-12-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"19366d88c352929b6d739cfa6f28bd187663b060a80af1a091ead7b6d2c177ad","result":{"confidence_high":33.38737846916739,"confidence_low":29.553615319652483,"sample_count":null,"score":31.470496894409937,"score_display":"31.47","source_stated_rank":null},"run_fingerprint":"e89fed9f61e54ecafae4ba8345086f33875265b2b95797e20a375c077127bd08","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-live","metric":"resolved_rate","name":"SWE-bench-Live · Lite","release_id":"release_9d2de9d5be7e9822af19f7b3","split_or_window":"lite-300-python","unit":"percent","version":"SWE-bench-Live Lite"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":338,"at":"2025-11-01","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:59Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-01","status":"stale"},"measurement_id":"349a4f14bb7c7d5851716d39c47c02d5cae18fb5dfa901c2b34e49084c6459c1","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 300 scored items with a 95 % Wilson interval.","leaderboard_script_url":"https://swe-bench-live.github.io/leaderboard.js","publication_status":"cleared-with-attribution","resolved_instances":73,"submission_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"SWE-agent","report_file":"reports-0605.jsonl","report_set":"lite","source_model_label":"GPT5-Thinking (Medium)"},"run_count":null,"scaffold":"SWE-agent","tools_allowed":"agent-specific coding tools in the official SWE-bench-Live harness"},"protocol_fingerprint":"ce57f73b26c759015b95b2efecef9bc95567cf2508fb51e1991b2009aa8dc27c","result":{"confidence_high":29.493618,"confidence_low":19.822078,"sample_count":300,"score":24.333333333333332,"score_display":"24.3","source_stated_rank":null},"run_fingerprint":"e9c7e94784c1e0d0a745d68a1d9cc7ffe24bd7c111b85ef8da6e884118dea6a8","source":{"content_hash":"e0226b6333c547885c15c6d53074d0ad5e2a7b99db2d6b54640d7d6537c242b4","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-09-30T18:53:18Z","homepage_url":"https://swe-bench-live.github.io/","id":"swe-bench-live","last_fetched_at":"2026-10-05T12:33:28Z","license":"Benchmark and harness MIT; aggregate leaderboard results redistributable with attribution (maintainers' confirmation 2026-09-02)","locator":null,"name":"SWE-bench-Live · Lite","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-bench-live.github.io/reports-0605.jsonl"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_80aa3dbf50108533097a37f6","split_or_window":"public-731","unit":"percent","version":"Scale public 731-task leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":312,"at":"2025-11-26T20:57:18Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-30T04:01:28Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-26T20:57:18Z","status":"stale"},"measurement_id":"f6646f2bce1e28d17bb808da74133a385ebade78da2ca333cc4b237ef545a4f2","metric_details":{"confidence_interval_half_width":3.49,"max_score":66.538,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gpt-5-2025-08-07 (High); createdAt=2025-11-26T20:57:18Z","source_repository":"https://github.com/scaleapi/SWE-bench_Pro-os","source_row_created_at":"2025-11-26T20:57:18Z"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-undisclosed-harness","comparison_warning_code":null,"judge":null,"run_config":{"agent_label_disclosed":false,"protocol_owner":"Scale AI Labs","public_task_count":731,"source_model_label":"gpt-5-2025-08-07 (High)","source_reasoning_label":"high","source_transport_sha256":"dcb96366a3abc8bb2a13886bbc1f931c8a2366fef5bf0ac5f3a943210f29f576"},"run_count":null,"scaffold":"submission-specific","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"488bbcf7575b92b6fa852b7e3ad33219335ac10a423a34f3c8dd07106a350e20","result":{"confidence_high":45.27,"confidence_low":38.29,"sample_count":731,"score":41.78,"score_display":"41.8","source_stated_rank":10},"run_fingerprint":"83188ef608474021c6aa56a73a7ff4e921ce70f7345fff3d3e7182ca9807842f","source":{"content_hash":"42dbde756a2514c54097aab1b5198fcd59bdb07a2903a38df38f64e33246c827","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-08-30T04:01:28Z","homepage_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public","id":"scale-swe-bench-pro","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with attribution; benchmark repository terms apply","locator":"embedded leaderboard row: model=gpt-5-2025-08-07 (High); createdAt=2025-11-26T20:57:18Z","name":"Scale AI Labs · SWE-bench Pro Public","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/swe_bench_pro_public"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-undisclosed-harness","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d1306e1da22ad074f1d1dec799fc177f7b667930475f5817e4123a80a2db89c9","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.063751,"date_is_proxy":true,"latency_seconds":150.441,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.974},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5-2025-08-07"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"fb734736410c54d50434d8f7d2380ce4c29215f56678a0df2395e67d6d16a6ee","result":{"confidence_high":87.82004,"confidence_low":84.00196,"sample_count":null,"score":85.911,"score_display":"85.9","source_stated_rank":28},"run_fingerprint":"aac8c06a9bbb8ba7382f09e45135c2c5c4dcd65c71234f7343be89ca7a4a9333","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"49787da5240b12a83566b632c34efa64bcc21be4106252c506b5e30918bbc7ca","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5 (high)","source_effort":null,"source_model_version":"gpt-5-2025-08-07_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"32272ea09707fc3d9f507d2a974dc3a1e1341ba5bad777b84934e9c5ffb81544","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.939814814814795,"score_display":"42.9","source_stated_rank":134},"run_fingerprint":"27bddfcfe8b832eed1967ca9b57b48c9f3c00d0534826ba57ef89a4652f8add9","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6523934a0b982f87b4855bad0c8479179e8c65fe6851d1da61c06e1211b1230d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.244948","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5-thinking","source_effort":null,"source_model_version":"gpt-5-2025-08-07_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"6191c7aa1febff3ba851da65081a2262db7f74a046aa23dc50284fe9550ceeb2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1162.45,"score_display":"1162.45","source_stated_rank":39},"run_fingerprint":"ea0ab15b17aa7b054227209d2f300ae32c31bd7604546b2a57afac57b839cfa5","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"64546890af3c74c7fd913e0ec8f50fcc9c837db96912accdbb0cb6cad6fae3d9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.043744","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5","source_effort":null,"source_model_version":"gpt-5-2025-08-07_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"4787e7add4e777c54c43b63fc78813ee639bcf075bf273411d11c96feda579d1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":807.65,"score_display":"807.65","source_stated_rank":64},"run_fingerprint":"a0a1b5d98c153d1f3905a597fe4c7ecede83497b3c95aaff42bde4f06dd7db70","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_3a0aafd1beceb97baedbcf78","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9225b2bcd7dee027d572c0269e18788825a57a2a4a3030b92126483032376aa0","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.525167,"date_is_proxy":true,"latency_seconds":1852.149,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.409},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5-2025-08-07"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"ebd7ac6b9a0d95694e85cf94c20a9a38a00c3c68b13399b90af8cdd66c9c9c9c","result":{"confidence_high":26.769640000000003,"confidence_low":13.406360000000001,"sample_count":null,"score":20.088,"score_display":"20.1","source_stated_rank":76},"run_fingerprint":"7201cb3baddba69ecfcaa0483c02d6f2ae7d25e20d6d853414b374f5194fad4e","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"efed06c23d5ee2273bdea5fcdef59ed5f93d0cc235688d878fd59a173c2093d9","metric_details":{"license":"Proprietary","variance":62.655788435046375},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5-medium","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1146113f27955fd60edb35c1926815de8d17742dbff1c47808cd4048536f8253","result":{"confidence_high":1432.5752821817869,"confidence_low":1401.5469318452735,"sample_count":3091,"score":1417.06110701353,"score_display":"1417","source_stated_rank":79},"run_fingerprint":"8ba3f9e423d491256670f93b292b24c9dde3ab93a8991f98d8c19592cae41d0c","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"136ff9a4d5b144342a3d9adc9f9c07697075029fc3f227b096ac3e6ba6be6868","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5-2025-08-07_medium","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"1567bc4d65c58b549fd4340e6d048e271a3e6bfd553f28a90bab7baf51831411","result":{"confidence_high":1435.27,"confidence_low":1403.78,"sample_count":3032,"score":1419.52,"score_display":"1419.52","source_stated_rank":78},"run_fingerprint":"c2e50f73d8150578be2b0c57c4c44bd76fd40416f903ea015bec916ba9c5f2d4","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a6fa54ec8908e72265e9e4eb4a20801934595b834cfae918b9cbf3a55b0c806b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5-2025-08-07_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"71f9acc41d93b97374a47997dbf2501842a44fc409d7d03676a3bb16355a0584","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.70000000000001,"score_display":"60.7","source_stated_rank":49},"run_fingerprint":"27985a723765d14d7122758fb22affd4b70676b34170d3c6018bd815b17e230a","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":335,"at":"2025-11-04","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-04","status":"stale"},"measurement_id":"d05a6de716f69f63a6c59a8ecbf3951d6b96f679e2f4e178baab7d933b663063","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"standard_error_points":2.9000000000000004,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gpt-5-2025-08-07_medium","source_row_date":"2025-11-04","source_scaffold":"Codex CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Codex CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"8474ce0c7801719c2dc058d6b23e9b023faf24739b2b39bf0756809c66430a81","result":{"confidence_high":55.284,"confidence_low":43.916000000000004,"sample_count":89,"score":49.6,"score_display":"49.6","source_stated_rank":74},"run_fingerprint":"f2cb4e9000ad8dd1a1db7c860e2ed6f54c3fcdc4299a3581f4b9bdac3eb44ed6","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":336,"at":"2025-11-03","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-03","status":"stale"},"measurement_id":"8e7370d0b16506ca6eb11c8173e70f87723963544137f2995698d2c3b2b36368","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"standard_error_points":2.9000000000000004,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gpt-5-2025-08-07_medium","source_row_date":"2025-11-03","source_scaffold":"Mini-SWE-Agent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Mini-SWE-Agent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"dbd25406a0a07840908e9ec0efb5b670f1a7e67cb6cc5c9cf0c5a3052f1ead4d","result":{"confidence_high":39.584,"confidence_low":28.216000000000005,"sample_count":89,"score":33.900000000000006,"score_display":"33.9","source_stated_rank":126},"run_fingerprint":"ca7fd01fa2dfb064f461184b5c0bab979c4b6479032bf7c68e64557cd4ce69f3","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":337,"at":"2025-11-02","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-02","status":"stale"},"measurement_id":"9bbaadacd9dc0e7908bd1a4436a36c603a1647f01708cba5bd5517a88be50da4","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":"Medium thinking used in other GPT-5 evaluations, assuming the same here.","standard_error_points":3.0,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gpt-5-2025-08-07_medium","source_row_date":"2025-11-02","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"f7284cb1e512e3d5cf2b6cbead6aab12c46ac0adc3073f94ddd4ec41d33dcab6","result":{"confidence_high":49.68,"confidence_low":37.919999999999995,"sample_count":89,"score":43.8,"score_display":"43.8","source_stated_rank":84},"run_fingerprint":"7fd904d39aaa7ac3c70f9fc0d7b8eb175290d8a1e9ca106b73aa68c7ce1584ba","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":339,"at":"2025-10-31","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-10-31","status":"stale"},"measurement_id":"bbf865937d00a0e62355be3db7092fa8eec3f06fc92ef072c1ddcdf3d20d2478","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"standard_error_points":3.1,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gpt-5-2025-08-07_medium","source_row_date":"2025-10-31","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"b49b63b968c75275fcc685599b7f0e4f9cdb4596adb07f67c9f407b9dacebb54","result":{"confidence_high":41.275999999999996,"confidence_low":29.123999999999995,"sample_count":89,"score":35.199999999999996,"score_display":"35.2","source_stated_rank":117},"run_fingerprint":"7bef28c52318523264fe7d133c09510554dc3e74244beda01976df63bbb50de1","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"b145d48928b98d2f52dc10580099a43cb213b57f38822e264d7b8da8ef660b75","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.33","mean_standard_error":"3.0","model_release_date":"2025-08-07","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.9,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5","source_effort":null,"source_model_version":"gpt-5-2025-08-07_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"c1d854957ac50774e1ffdfeeb6300240986006f71e614b934312b772ca93eb40","result":{"confidence_high":23.984,"confidence_low":12.616,"sample_count":480,"score":18.3,"score_display":"18.3","source_stated_rank":39},"run_fingerprint":"7805cc75d66d1ea4e172779fd2a5295cf38c9d054e525ed4bac726dbbd5777bf","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"56079051e891d8d317d54b43eedba8bf07f2b15c0dc8cfadfed2e2987cb9bb7f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.33","mean_standard_error":"3.0","model_release_date":"2025-08-07","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.9,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5","source_effort":null,"source_model_version":"gpt-5-2025-08-07_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"906d1e5ad41d720d8e542bd244b65d7b68e52c9e879b1087c7c5e6f9c9459b28","result":{"confidence_high":23.984,"confidence_low":12.616,"sample_count":480,"score":18.3,"score_display":"18.3","source_stated_rank":39},"run_fingerprint":"1a1d63f775dbb2488e837e5742971eddcbce92a216e07c0c6ea992e06a88fe8b","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d88f5799bf95e4b1ceb57033","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"94ab7b929be4375e1164629b49c626b61ee9208f238e0050d862101f7d6b7201","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":400000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2025-08-07","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · GPT-5 (High) · gdpval"},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5","upstream_model_label":"GPT-5 (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"263832cc4a81f2af93970ad33935009c4315e10135a9e4295a682d628f82c7f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":905.76,"score_display":"906","source_stated_rank":null},"run_fingerprint":"bc8f486ad74355d081010cf5e120a0126e12e612b878ba3d2f6dac68698cded5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5 (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_95a163c92c81560a416e6d07","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval","metric":"win_rate","name":"GDPval","release_id":"release_730527fa346470703bd19e21","split_or_window":"expert-comparison-win-rate","unit":"percent","version":"GDPval official leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:05Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:05Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ebf8b73860d4784f1ce1f70ef8ef3d05fb73dedb007c56937f78c3fdbd791012","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-07","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5","source_effort":null,"source_model_version":"gpt-5-2025-08-07_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenAI GDPval expert-comparison protocol","tools_allowed":"as reported by the GDPval leaderboard"},"protocol_fingerprint":"4ea479ba4b62f3c7f3adcca3d3e064f21bfadff49718e6dec2bd9ce82098ce9f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.8,"score_display":"34.8","source_stated_rank":6},"run_fingerprint":"3a3638c543e0460c18dddf1ab688585e1ca3419f1461421321bb0ec73786b7ed","source":{"content_hash":"d4eac25e0863d96ee45cc545fc896e5667d7022da8b3bc3902533ed87fa8b836","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gdpval","id":"epoch-gdpval","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GDPval","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gdpval"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifbench","metric":"inst_level_strict_acc","name":"IFBench","release_id":"release_4adf348ac2f76539ed7ccb10","split_or_window":"ifbench-test-300-prompts-344-instructions","unit":"percent","version":"swallow-evaluation-instruct"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-04T14:29:18Z","observed_at":"2026-10-04T14:29:18Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"98f3233b2ca9c634372252fad47b715ba0bbf8e96bf00101669484d337220c47","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","instruction_count":344,"model_release_date":"2025-08-07","prompt_count":300},"model":{"id":"openai/gpt-5","name":"GPT 5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"rule-based instruction checkers (lighteval 0.11.0 with Swallow fixes)","run_config":{"decoding":"temperature 0.6, top-p 0.95","leaderboard_revision":"c58a13a852eba061d639af7caf4ddbdff33b36e3","protocol":"Swallow LLM Leaderboard: swallow-evaluation-instruct (lighteval), allenai/IFBench_test, instruction-level strict accuracy; the final answer without the thinking part","swallow_model_id":"gpt-5-2025-08-07","swallow_reasoning":"on (medium)"},"run_count":null,"scaffold":"swallow-evaluation-instruct (lighteval), swallow|ifbench_singleturn","tools_allowed":"none"},"protocol_fingerprint":"05873542cd642cd62613c324841b4bee4e7d770f0b13c2c614fd95d84790cd2d","result":{"confidence_high":null,"confidence_low":null,"sample_count":300,"score":72.1,"score_display":"72.1","source_stated_rank":null},"run_fingerprint":"f5f78b8b973030a9c85a60f7ce6af2289cd59bc0f3e7ff75f2d00d6cf4164d6f","source":{"content_hash":"fffdc5bb4b3de214a8a4ce885cf286e13a4b1488d9a4a0791e5918d1918b74cb","fetched_at":"2026-10-04T14:29:18Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://swallow-llm.github.io/leaderboard/index-post.en.html","id":"swallow-llm-leaderboard","last_fetched_at":"2026-10-04T14:29:18Z","license":"Leaderboard content CC-BY-4.0 (Swallow LLM Team, README of swallow-llm/leaderboard)","locator":null,"name":"Swallow LLM Leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/swallow-llm/leaderboard/c58a13a852eba061d639af7caf4ddbdff33b36e3/swallow_leaderboard_post_en.js"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_a8c935191f53cb042adf83f1","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6b7d8f492a92502355d9a9aeaf41618a0cc2202674da4347c75c205dfe92b8c8","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-07"},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":false,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GLM-5.1; model release: 2026-04-07","source_accessibility":"Open weights (unrestricted)","source_model_name":"GLM-5.1","source_model_release_date":"2026-04-07","source_model_versions":["glm-5.1"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"890936c76971023f2cf9294c3b2b7112f79008c57aa6ea1473c55d1886337eae","result":{"confidence_high":151.61,"confidence_low":147.97,"sample_count":null,"score":149.9,"score_display":"149.90","source_stated_rank":52},"run_fingerprint":"6c8803738a5f7e76882ca1697b56411a4232550ed560bcbfe84ef32dd341a2d6","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dccc52ab99879471547d0fcb","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b83f315c50722fc87854f8c653e635e6cc3a64f07d6e72d57414e7a65bc78cb8","metric_details":{"license":"MIT","variance":3.5304047864283286},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1468.2481179975305,"confidence_low":1460.8828197394976,"sample_count":58358,"score":1464.565468868514,"score_display":"1465","source_stated_rank":55},"run_fingerprint":"10a450981b7cec0101dbfc946239b35cf60b4d9fbe0bc532f6411d2fd645635a","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4f6d1e7644946448f8c862a5","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":181,"at":"2026-04-07","basis":"evaluation_at","evaluated_at":"2026-04-07","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"66ec7986f4c10236c40b538bde7f1b2212821b7bb5fe9064ca96432e688c301c","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/zai-glm-5-1","source_model":"GLM 5.1"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ee783082ec47e7a371a58d8a675a96b1f52ecfbbc63322f29a55942e98f28297","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.63,"score_display":"25.63","source_stated_rank":32},"run_fingerprint":"927de019e81ab3926f263cb59e3050d0e831664ddece7352561c3ce0221d58e1","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_431e0630dbeb9073dc747d67","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":181,"at":"2026-04-07","basis":"evaluation_at","evaluated_at":"2026-04-07","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ea007127dd467804b3e3ff9030de90973fc84bce91d252ed1b88a3cffb863452","metric_details":{"modality_lane":"not reported","notes":null,"num_parameters":753864139008,"pull_request":null,"result_file_url":"https://huggingface.co/zai-org/GLM-5.1/resolve/main/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/zai-org/GLM-5.1"}},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.0,"score_display":"31","source_stated_rank":38},"run_fingerprint":"be11b60fc19e0560a32134566a96e779770e0b94a79d5397eb8cd71b73d9ae52","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_11d0dbb9b5c83946999afc1d","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":181,"at":"2026-04-07","basis":"evaluation_at","evaluated_at":"2026-04-07","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c0ce540ee89ca9cb789d20c9f91184e60da883a2aee972eeb2118493932e742f","metric_details":{"modality_lane":"not reported","notes":"With tools","num_parameters":753864139008,"pull_request":null,"result_file_url":"https://huggingface.co/zai-org/GLM-5.1/resolve/main/.eval_results/hle_with_tools.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/zai-org/GLM-5.1"}},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.3,"score_display":"52.3","source_stated_rank":11},"run_fingerprint":"8510568fb5145814f6819878f82e6f15e3f020b88c0b135379d894fa7122ad81","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_11d0dbb9b5c83946999afc1d","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:36:21.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:36:21.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"01f6e3240327346fe18e8703ad7b4e4b67ef006c3f5f1653b7f2032a3a2161dd","metric_details":{"best_score_across_scorers":"0.34","model_release_date":"2026-04-07","stderr":1.4987482264363972},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"R2fMFAtRXah864hMifdQGV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FR2fMFAtRXah864hMifdQGV.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/R2fMFAtRXah864hMifdQGV.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"c82c0b1cfbc31d6ae84f5ace46374d5aec7d3ac6893533ed29941bee1c4deb8e","result":{"confidence_high":36.937546523815335,"confidence_low":31.06245347618466,"sample_count":1000,"score":34.0,"score_display":"34.0","source_stated_rank":55},"run_fingerprint":"5fba827f219cfc5999982dc42a27a978acd2dca1de5b61612451fd3060817c9e","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0284afb597a0b16d0dfbcb57","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b1bf2e5bc4fbf54f2ee74e23cc009585201adb414e7b66a6db86007c65c57d64","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.044897,"date_is_proxy":true,"latency_seconds":249.704,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.821},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":85.787,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":null,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"zai/glm-5.1","zero_shot_accuracy":83.249},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"ae2a39a4845fa7c0fe32a055db811052e4451cb26a9d8272b314cc86fa5ab35b","result":{"confidence_high":88.08716,"confidence_low":80.94884,"sample_count":396,"score":84.518,"score_display":"84.52","source_stated_rank":52},"run_fingerprint":"6645b6edafeea769f65ec5948b5efd3051f59e2c4d69a8f521a004b752af757a","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b1fc7c73685381c163b9d8b8","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":55,"at":"2026-08-10T17:44:34.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:44:34.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2129abdc1b8592eb58d8ce6a65e0aa9e7dd6c74bc944cf7b422cf940dd63fcc4","metric_details":{"best_score_across_scorers":"0.898989898989899","model_release_date":"2026-04-07","stderr":2.1469735576055355},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CBbeMKRRSeRsXnvSfn7Png","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCBbeMKRRSeRsXnvSfn7Png.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/CBbeMKRRSeRsXnvSfn7Png.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e061ac18239aeebf5b200c2aa1f1bfcb2c16e903ca5c89fece6d5d75900c3d75","result":{"confidence_high":94.10705807189674,"confidence_low":85.69092172608305,"sample_count":null,"score":89.8989898989899,"score_display":"89.9","source_stated_rank":46},"run_fingerprint":"4774c60642e0f5178b98befb1b1d137d804b42023a9f0527198f2d179fdc2c47","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_981ea0f2e4bb744fe8177226","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"240537baa817e3a3bd70f894ea11dd8d237f8f4c8a91b6664ceb5b6f1960aa4f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.012383,"date_is_proxy":true,"latency_seconds":69.318,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.334},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":null,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"zai/glm-5.1"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f5ecca7b6343991ef892535468d1a6b64194afb11cb55d2714dde52cc5b9fbbe","result":{"confidence_high":87.55464,"confidence_low":86.24536,"sample_count":null,"score":86.9,"score_display":"86.9","source_stated_rank":37},"run_fingerprint":"bfca36e0155239915032be3b7c98bb1f7c6e127ee104af4928a702bdd58a0bca","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8758124509145d977399bb3b","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:44:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:44:52.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9cb7bf25a9ee062ebcee4c8d2f6137b2f315b34e71ab1c57cdaee6c72731cb98","metric_details":{"best_score_across_scorers":"0.9333333333333333","model_release_date":"2026-04-07","stderr":3.760507165451774},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"b7Lw9ExxAhVY3hdExoh4MP","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fb7Lw9ExxAhVY3hdExoh4MP.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/b7Lw9ExxAhVY3hdExoh4MP.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"392497471e5edbe75c8a86361b5bd275c44d0a931cbec936fb5c638af6cadaae","result":{"confidence_high":100.0,"confidence_low":85.96273928904785,"sample_count":45,"score":93.33333333333333,"score_display":"93.3","source_stated_rank":53},"run_fingerprint":"a1e366eb55995c1a33213da4ccda2bcb21be4b7e4141c51fa83b3773d5dc23b2","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b4a08efe49d38b527af1c6f5","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-29T11:06:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-29T11:06:22.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"98b1c53f39b38d4d2bd84a343626db94e65bf32017c030d146d94e86e22cd8e9","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.3684210526315789","leaderboard_private_problem_count":285,"model_release_date":"2026-04-07","public_example_count":10,"stderr":2.8623772537150214},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"Mhwme9odWJMzMjoYkW8r6B","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMhwme9odWJMzMjoYkW8r6B.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Mhwme9odWJMzMjoYkW8r6B.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GLM-5.1","thinking":null,"upstream_model_id":"glm-5.1","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"98dcfe9815fb35e05957550b906d55eb847952b2b3400aa5542360e7fec9708d","result":{"confidence_high":42.45236468043933,"confidence_low":31.23184584587645,"sample_count":285,"score":36.84210526315789,"score_display":"36.8","source_stated_rank":60},"run_fingerprint":"6c61d79992fe572d68f299cc3e317af49fad1e6599e0e1b3fc145b6ee0c11d3f","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_48e96ad44abb27180373f267","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-28T12:30:03.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T12:30:03.000Z","first_observed_at":"2026-09-01T00:20:16Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4e90df5f66d30162e22fed818a7844e6befa2fad4a819a5388f0efb1c55355cf","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.24912280701754386","leaderboard_private_problem_count":285,"model_release_date":"2026-04-07","public_example_count":10,"stderr":2.566445410232946},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"4hmmfwG9tgZJvKhL78nKzL","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4hmmfwG9tgZJvKhL78nKzL.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4hmmfwG9tgZJvKhL78nKzL.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"","thinking":null,"upstream_model_id":"glm-5.1_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"48e513b56c623e176b2eb64f084bd508a1a4cb99c20b7dbcbaa0a3c5019d41ad","result":{"confidence_high":29.94251370581096,"confidence_low":19.882047697697814,"sample_count":285,"score":24.912280701754387,"score_display":"24.9","source_stated_rank":76},"run_fingerprint":"c929cdac568bfa712253534b6b30e627ba564f0d11c18679825cf3b01a1496d6","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e86c77653aa4667f605803cc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d367db4f7c113458a6e9d6e5a7e414134a738c533657f8b5417c6fe2409602b4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-07","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.1 (Reasoning)","source_effort":null,"source_model_version":"glm-5.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c001b94d23a5145180ef0494950146d455b821d9c6ac0adb93a4a086462d965d","result":{"confidence_high":12.178661,"confidence_low":1.62787,"sample_count":71,"score":4.57142857142857,"score_display":"4.6","source_stated_rank":113},"run_fingerprint":"aa0a132bcaf296b2c1f8438857aaef80bb5e5a6e0cfcaab3eb0b24ccd2117ec9","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4527a4a9bd084290f3d0c928","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aef06d7684ad7d98202f3c6f99461374d38cfaf75d88a54605ac389d6bc78fca","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-07","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.1","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"4e1356ea08402efb38774ffe2ae9e2d6cec6e73c9c6b46736562da724a69ee60","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.1,"score_display":"55.1","source_stated_rank":42},"run_fingerprint":"f00999cb61a211e645e95c1b32ce7667335255bae4511eda408f614a4e1f50ed","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_843141e4bdc30672d535a46d","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:44:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:44:45.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"40a56709cd245ee818420c1ff0b2652a4cf40dd4216241c8678ca1ff8a88ae2c","metric_details":{"best_score_across_scorers":"0.19","model_release_date":"2026-04-07","stderr":3.9427724440366254},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"kmHbjBrwGgEKNr65JrXcGX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkmHbjBrwGgEKNr65JrXcGX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/kmHbjBrwGgEKNr65JrXcGX.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6ee4c07018c748932ba5d6b868aa459a1773742932c11be2c311becee91de6aa","result":{"confidence_high":26.727833990311787,"confidence_low":11.272166009688213,"sample_count":100,"score":19.0,"score_display":"19.0","source_stated_rank":99},"run_fingerprint":"24e24b282c1005f3d15b7d199e0fc9173821d298d713bb895cc687fc7c9270b4","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c0691b96512398ad384aacdc","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0440d95d50f3ae695c266790304fc9203aacddb34dc89ab48a120421bfc630e8","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.456603,"date_is_proxy":true,"latency_seconds":527.071,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.901},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":131072,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.1"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"0b01c10d49f156c85d867fc10258387e90327942cbf98b6db33106e83db94e22","result":{"confidence_high":80.12596,"confidence_low":72.67404,"sample_count":500,"score":76.4,"score_display":"76.4","source_stated_rank":37},"run_fingerprint":"fb89b9092e6442e70842b8978af8f3c67ae8290a394152da73e36eeded6245bc","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ecd426e90f3b64f9c038b350","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T10:07:59.494Z","basis":"evaluation_started_at","evaluated_at":"2026-05-15T10:07:59.494Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3b4e1959b77038d1ee3a88e002546cd1114118755c81661ca9ac020168770dec","metric_details":{"best_score_across_scorers":"0.7417","model_release_date":"2026-04-07","stderr":1.9900000000000002},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"glm-5.1-swe-bench-frankenstein","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"378fe31fb9b28af0a787e5940f596fd94d73606816de1198ca9dc4b797820d27","result":{"confidence_high":78.0704,"confidence_low":70.2696,"sample_count":484,"score":74.17,"score_display":"74.2","source_stated_rank":17},"run_fingerprint":"6ad4068aa8a10c556a23f8fdefb8de0ae724a5d45ced746895cc7bfdc78e8861","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8be4a07d552abc9ade9567a1","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_1f8553cc12d8235844f1a578","split_or_window":"2026-04-15/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-29T23:53:02Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"53bad1f8d9084c8f2f6208c2bd80f77cc837084939865ec6099ed7667b143920","metric_details":{"comparison_warning":"Official post-release SWE-rebench window; use only for same-window comparisons and never mix it into the canonical rolling-window rank.","comparison_warning_code":"swe-rebench-post-release-window","model_release_date":"2026-04-07","pass_at_5":64.1025641025641,"potential_contamination":false,"sem":2.205724427446827},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-post-release-window","comparison_warning_code":"swe-rebench-post-release-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","supplements_range_key":"1772323200000:1778803200000","upstream_model_id":"GLM-5.1__tools","window_from":"2026-04-15","window_status":"supplemental-post-release","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"adc23987b5b7dec5bcf4893f7ec22af1b131cb7476d633bfc720fba4b5732bb6","result":{"confidence_high":50.98988654446244,"confidence_low":42.343446788870885,"sample_count":null,"score":46.666666666666664,"score_display":"46.67","source_stated_rank":null},"run_fingerprint":"28bd29daa57451d9be9db498c142c26167d65448a40340d04b936d5d06775743","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b8e62d13cae9686240097b66","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-post-release-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"d0643ee1502267b6db81010174e601fcd511c29c3fbd71649c32a0a7ddaa1b86","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-04-07","pass_at_5":65.45454545454545,"potential_contamination":true,"sem":0.9270944570168703},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","potential_contamination"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"GLM-5.1__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"84265b74fdb8984a2c9eec93d651ca3ab644ea5df2ef4cd31ce44c3f5978dc49","result":{"confidence_high":52.54437786302579,"confidence_low":48.91016759151965,"sample_count":null,"score":50.72727272727272,"score_display":"50.73","source_stated_rank":null},"run_fingerprint":"725fe81d0676d8ba6903d9f67d0d3e8c1d550b5be063d239c5bdf69e18d344ea","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b8e62d13cae9686240097b66","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_aa29eb18bdb9e43cee539f4d","split_or_window":"zai-openhands-tailored-prompt","unit":"percent","version":"SWE-bench Pro · Z.ai OpenHands vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:19Z","last_confirmed_at":"2026-09-16T06:32:03Z","observed_at":"2026-09-16T06:32:03Z","source_freshness":"expired","source_observation_age_hours":463.9,"source_observation_status":"overdue","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"31c8ab72a679addad4a4ec8ecb305eb21b3cef6117e8bbd19591cf706555838f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Launch benchmark table · Coding · SWE-bench Pro · GLM-5.1 column; Evaluation details · SWE-Bench Pro"},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Z.ai","evidence_verified":true,"freshness_proxy":true,"protocol":"OpenHands with a tailored instruction prompt; temperature 1; top_p 1; max_new_tokens 32k; 400K context; exact task revision, task count, run count and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-06-16","source_content_hash_method":"raw_bytes","source_content_sha256":"a9e8c2b6f34717d69e3a0aa26bb117256a4d8c95bd299910c2a693000ee88fe8","source_published_at":"2026-06-16","tools":"OpenHands coding-agent tools"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"OpenHands coding-agent tools"},"protocol_fingerprint":"99fc4cef8010e0643d87d80ddb87946b7222f6c359dec21c878cd22d0e12a155","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.4,"score_display":"58.4","source_stated_rank":null},"run_fingerprint":"4dce714a1bba3ffdb12d31ed90c3a6f94050ba79d5124bb5707c9d7216ef070c","source":{"content_hash":"2b25311712ee2dd95319406ca612fd13303e81cf3f2819759a111412e6dec532","fetched_at":"2026-09-16T06:32:03Z","first_fetched_at":"2026-09-02T12:19:34Z","homepage_url":"https://z.ai/blog/glm-5.2","id":"zai-glm-5-2-report","last_fetched_at":"2026-09-16T06:32:03Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Launch benchmark table · Coding · SWE-bench Pro · GLM-5.1 column; Evaluation details · SWE-Bench Pro","name":"Z.ai · GLM-5.2 launch report","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://z.ai/blog/glm-5.2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0e0010bf2c9afedaf89648d","provider_config":{"source_id":"zai-glm-5-2-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"132179068e88d6e6aac889a2e790bf88bd306e164dfd5238fb0f19ed39ac5253","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.045983,"date_is_proxy":true,"latency_seconds":226.424,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.061},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":null,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"zai/glm-5.1"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"fa11f4f1c96665a81195ec87cea8ef0279792141b30964aa4619dd3890124f5a","result":{"confidence_high":83.45956,"confidence_low":79.30044,"sample_count":null,"score":81.38,"score_display":"81.4","source_stated_rank":63},"run_fingerprint":"eb20b05cec208df7f7427ff8de5d10d6cfb84999a32587bf69762321ec0676a2","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d4071b8077bf3ed5a584e7a0","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"40e5af62da624f456e9c4a255d7a86f34a727cf5c1a780fbd67a8ec0f49457f0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-07","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.1 (Reasoning)","source_effort":null,"source_model_version":"glm-5.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"255b918a3964d48cf69d2ac2b7c2427870abb9538b96a76deff23a505934f694","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.75,"score_display":"43.8","source_stated_rank":128},"run_fingerprint":"90941ac3cf96acdce6cba0d856184d95b4deadab3b2caae50802ca40dd1679aa","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1a321a729524c98a8dd06766","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e296211307a7e6dd253e33c0eb548586877c9326308d9e14a1668e022afad421","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.12937","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-07","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5.1","source_effort":null,"source_model_version":"glm-5.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"87ad9863a296e4d0ca4c603bd65bf083b55cd7df19331b42c8fbfb054df34ad1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":887.1,"score_display":"887.10","source_stated_rank":60},"run_fingerprint":"5d30aad38644aecbd03105d281c11ef6a3824ccebb060c8c3a71ab370b6b75d8","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_30b8a704bb9675a95ba357fd","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"13bef6aad46095066383207dd7a733efe56d3db828625d70c2a4935d2441f780","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.894911,"date_is_proxy":true,"latency_seconds":2014.732,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.553},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":131072,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.1"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"9101aeff2479fec85b35f33b70fa6b4eae507b87e0cf49517bdac9a977012a90","result":{"confidence_high":40.37988,"confidence_low":22.53212,"sample_count":null,"score":31.456,"score_display":"31.5","source_stated_rank":66},"run_fingerprint":"c066e568371e700e3cd5244679cab63a0fca48466268be75b7a3501246096672","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a3ad892dee5035b3e24ce86c","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"50d843ae2bb3f996ede39f6b7e31b109a91253a435559bfb249babc08e831a83","metric_details":{"license":"MIT","variance":11.858738508249699},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"glm-5.1","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"77e1caf3601a28c4dd7ed4ab02b25561be5aea50385508c3e2dab2c7a86dfd20","result":{"confidence_high":1515.2448719106033,"confidence_low":1501.7460044551726,"sample_count":10797,"score":1508.495438182888,"score_display":"1508","source_stated_rank":52},"run_fingerprint":"43ff870d4717455281c840e9534031cfebfca430db3a177eab291514136d0dce","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c3270c0606da8408f8bddd6a","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ffe829ed74847a57f414198aa28277baf91cb7c9f06cc18b8d528f54510b9f30","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-07","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.1","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"96f453babb78bf1229209fb307f9e12b57f67a676dcd3007649d1ced48653dad","result":{"confidence_high":1515.2,"confidence_low":1501.38,"sample_count":10220,"score":1508.29,"score_display":"1508.29","source_stated_rank":50},"run_fingerprint":"2f09512eb052a10d2216f9fecc6b4caa6e731a502521f4e4eb1d07d36bb8d4de","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ce5d20a4bae1fb8d98230078","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"53600c87b058fb04d969bc58d59f4c354d0325accffa3f3ebf5cf57a0a72bef8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-07","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"c0b0aa6e7a0042316f3fbcf1b37e0959dbd6fc34c9e041d04a2c348019e00d3f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.10000000000001,"score_display":"57.1","source_stated_rank":56},"run_fingerprint":"0a4ff0469367851e1090cd7c7f243a4297aac7fe46c01a997152e4f0f83b3c74","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3d8decf657d2d47a2ddf04aa","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"71fc41cf96b4ab6ea0bd54d1bc3b5dbf87eeefa41734ef5b28cdff3578dfd033","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.232386,"date_is_proxy":true,"latency_seconds":940.943,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.35},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":131072,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5.1"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"c7ff7246e344fb75bc24c3622132a47853217273e80c391e86b37430bc2db017","result":{"confidence_high":59.575,"confidence_low":54.283,"sample_count":null,"score":56.929,"score_display":"56.9","source_stated_rank":48},"run_fingerprint":"a6b736446c95b2b3009c68b169f537d5bd753c6ef67841275c3206edd79ae64d","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3e84dc8d0f289a1a0746e436","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":291,"at":"2025-12-17T17:20:04Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-17T17:20:04Z","status":"stale"},"measurement_id":"b6c8f4365ef05b1f9ba7893c42c6810bbcc2eb7217f575dab9655ad4b3830a9b","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.7,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=glm-5p1; createdAt=2025-12-17T17:20:04Z","source_row_created_at":"2025-12-17T17:20:04Z","task_pass_coverage_threshold_percent":75},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"glm-5p1","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"042fc987bed027cdeb06d3955f8215e4fd06001d75ad8aa46277ad0f71d3d62d","result":{"confidence_high":78.3,"confidence_low":72.89999999999999,"sample_count":1000,"score":75.6,"score_display":"75.6","source_stated_rank":3},"run_fingerprint":"ac73ac88b34c3f73383ff61b2ff372fe016816c3534bf60ff4b16fd981c76cda","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=glm-5p1; createdAt=2025-12-17T17:20:04Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ad3fab65029bdb5cc6f0ca02","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"9f892f7e5de52b1dda5e89136236d83cc2ad0e8a37423b0fd422b0e97bf5752e","metric_details":{"expected_trials_per_mode":261,"normalized_gain":38.1,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":25.7},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":67.2,"confidence_low":49.599999999999994,"sample_count":87,"score":58.4,"score_display":"58.4","source_stated_rank":5},"run_fingerprint":"deb148730c332174819980f9b06a681381a2cf295d78afeafd7e77c680984db8","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7b5a3c7d436cb995e1ba2cae","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"b2b6485ee9a1a58ed16dc7eb9e93c60f1fba63ce749550bb4d77f162a2551736","metric_details":{"expected_trials_per_mode":261,"normalized_gain":38.1,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":25.7},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":41.2,"confidence_low":24.200000000000003,"sample_count":87,"score":32.7,"score_display":"32.7","source_stated_rank":12},"run_fingerprint":"f73d0cb5ea39d94764345cbf6d13155a13f81859bb13678700f84fefdc25a77c","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7b5a3c7d436cb995e1ba2cae","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8be3143dabb1e3529889192c40846058a2f0005e306dd096b27b080324981b9d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.568","mean_standard_error":"4.6","model_release_date":"2026-04-07","notes":null,"standard_error_points":540.0,"upstream_source_url":null},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.1","source_effort":null,"source_model_version":"glm-5.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"c387655e8b82664bcf3e899a94df5232307cbe6ae09f289901ac9a8ba3369a2f","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":40.9,"score_display":"40.9","source_stated_rank":32},"run_fingerprint":"c39bb4083b950ab8561d6505ec95b183b692b29241a72f7df09434f77be2c27a","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8040d78652f9243c2fd1388b","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"91cd508a5670660029d8f9112e7d35cc975d039c96f65da748b59903f34ac144","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-07","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5.1","source_effort":null,"source_model_version":"glm-5.1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"9a9e32b0068f5fb9b0b566ae38d4d1d61625904821eac60cb0d101ea97d1ad67","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5634.406,"score_display":"5634.41","source_stated_rank":21},"run_fingerprint":"7cd35a1fe4e9f6d45d774da7eeb1b1ae41579c0ba82d58bf31cef3b3a0234763","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7ec5cca37db5e5d05e3414c7","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dfaa8258597689f3d442acd4f38621d0ba5aeefe4ffcc6d05eba0a311784ad62","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":200000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-07","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · GLM-5.1 (Reasoning) · gdpval"},"model":{"id":"zai/glm-5-1","name":"GLM 5.1","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"glm-5-1","upstream_model_label":"GLM-5.1 (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"33331422d66c6c1f212bc626f3ada790d68caca1ede21fec40f57266b89efbca","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1103.36,"score_display":"1103","source_stated_rank":null},"run_fingerprint":"c8784df3dcff5f29cbb94ecf53e3eb2e3cc5c5f88773f5d842f1b3ac237516d9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GLM-5.1 (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_41edc6ecadbdb6a4fe4eb5b0","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"661b48504171497243913b98420dd139ba26d4f601f609084fa2514490298f5d","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-11-13"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.1; model release: 2025-11-13","source_accessibility":"API access","source_model_name":"GPT-5.1","source_model_release_date":"2025-11-13","source_model_versions":["gpt-5.1-2025-11-13_high","gpt-5.1-2025-11-13_low","gpt-5.1-2025-11-13_medium","gpt-5.1-2025-11-13_none","gpt-5.1-2025-11-13_unknown"],"source_reasoning_efforts":["off","low","medium","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6337416ef138f93858f7b276c33d41dc84a6f10de5a2b95aed77a52132be4ba2","result":{"confidence_high":151.12,"confidence_low":148.28,"sample_count":null,"score":149.65,"score_display":"149.65","source_stated_rank":53},"run_fingerprint":"0c7d65b6930e400730f8644337105514ec6e353b2ee27f3304eff6c3d6f6e057","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_feeea085ab7763bb52bc3cc1","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"97623f267ddf33ce9910fa416806c64337d51d2dd7deafb1f7222cd645e71ed0","metric_details":{"license":"Proprietary","variance":3.5919153295079704},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1459.487242084938,"confidence_low":1452.0580577890075,"sample_count":41577,"score":1455.7726499369728,"score_display":"1456","source_stated_rank":69},"run_fingerprint":"c7f9975387a30af7a12ebb0d9a2928b3445ee5bfa8f40f6dfcea27673a5c94d1","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5f7f80987a407ba66ba6ca24a79b4eddf806b3760627a12d4ae71fd63e812ea8","metric_details":{"license":"Proprietary","variance":3.3710137384593866},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1442.1043381315887,"confidence_low":1434.9072246934704,"sample_count":44298,"score":1438.5057814125296,"score_display":"1439","source_stated_rank":97},"run_fingerprint":"68f5edbe0381180dc0495c953267de744ca8736203ec0333a5013c12d7202216","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_53b7f349317a61854d78ebda","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":312,"at":"2025-11-26T20:24:19Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-26T20:24:19Z","status":"stale"},"measurement_id":"2c5afebc4239463132bff69dd9dbb0a5e84ed45e8ef0108ecb410fb91e16c0f6","metric_details":{"calibration_error":55.0,"confidence_interval_half_width":1.67,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=gpt-5.1-thinking; createdAt=2025-11-26T20:24:19Z","source_row_created_at":"2025-11-26T20:24:19Z"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"gpt-5.1-thinking","source_reasoning_label":"thinking-unspecified","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"2945049d8df09567b063624927103ea87121db08ed242ab08aec9bd2724812da","result":{"confidence_high":25.35,"confidence_low":22.009999999999998,"sample_count":2500,"score":23.68,"score_display":"23.68","source_stated_rank":14},"run_fingerprint":"ef07ae1d0dbb1bba4fbe615f797b349de72802dccb71e11fb5b075387fde2d3b","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=gpt-5.1-thinking; createdAt=2025-11-26T20:24:19Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e92c503ca7d7c630bd73c098","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":327,"at":"2025-11-12","basis":"evaluation_at","evaluated_at":"2025-11-12","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"9a72ae21e3bb7efe778eec40d4e3deee4a20541d4da59db7aa1a3c27ada57591","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/openai-gpt-5-1","source_model":"GPT-5.1"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"81cc4e213594e68b2016cd368f5a2a2284b0e92a16781cc9d076d921bf6462cc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.2,"score_display":"27.2","source_stated_rank":29},"run_fingerprint":"4626d87ea55edef8cd8e97d75fcc8755fe6ce70a6161990e8c9e7d6c88b7fa1b","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_73e3f5b5ff32116dd0068c06","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:28:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:28:24.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"597bcaa6f5d0254b5b2292d2926b265101b4f97787aab2c31452630ac9541fb6","metric_details":{"best_score_across_scorers":"0.48","model_release_date":"2025-11-13","stderr":1.5806639423035178},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"QCLnRy69NBH9ip3VYtQneh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQCLnRy69NBH9ip3VYtQneh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/QCLnRy69NBH9ip3VYtQneh.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"5ad4fb8ad8c4948f2431ce9816a32c2e10e39c3605456ba246ae3d4dc99f004d","result":{"confidence_high":51.09810132691489,"confidence_low":44.90189867308511,"sample_count":1000,"score":48.0,"score_display":"48.0","source_stated_rank":31},"run_fingerprint":"5ce22edda06a5c55505f1930bf0755a290b4bc9b58ac8d35c504edc1afe67fb8","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":323,"at":"2025-11-16T00:00:00Z","basis":"evaluation_at","evaluated_at":"2025-11-16T00:00:00Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fb1230807946ffed558166dbaa293ed58ea3a0eeb0a7ee63f06bbfa0d62afba3","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gpt-5.1-2025-11-13","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"185830bc3c5dbcdc75ffe1da5362e6c52ca7239b270e624a46087e2f00f95f33","result":{"confidence_high":37.89568753796181,"confidence_low":31.94876805661921,"sample_count":null,"score":34.92222779729051,"score_display":"34.9","source_stated_rank":20},"run_fingerprint":"3401cf25d1e0142398d930136ebd5311f455eee28dfee1b96d46bf2c5fa87409","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a02ac21144bbf58e887b1177","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"294d064de37a10a914544ec96508d944de8d2b8f36783ebcf07771550b029654","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.074563,"date_is_proxy":true,"latency_seconds":85.708,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.711},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":86.364,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.1-2025-11-13","zero_shot_accuracy":86.869},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"87139f696a19cded45c2f4da5f216d12801c99cba1fbb87480ff3c1d41c1fda2","result":{"confidence_high":89.96956,"confidence_low":83.26244,"sample_count":396,"score":86.616,"score_display":"86.62","source_stated_rank":41},"run_fingerprint":"d404c0ceb8959c85e58fc9f27ba346ca9bfde0eb9206d135a82249ead7a88ce1","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":325,"at":"2025-11-13T19:52:15.039Z","basis":"evaluation_started_at","evaluated_at":"2025-11-13T19:52:15.039Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"83c2258d780924354c4437fcaaeadfbba38e2064d5d5ff8bccfa8eefef12272b","metric_details":{"best_score_across_scorers":"0.8762626262626263","model_release_date":"2025-11-13","stderr":1.8753682810775951},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CSVtMksEVjBNsGJGjW4GcQ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCSVtMksEVjBNsGJGjW4GcQ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/CSVtMksEVjBNsGJGjW4GcQ.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"1c2860e91b4cddb43d7e15b1494626c35eaca13faf48aa7c76177209b1f99bfd","result":{"confidence_high":91.30198445717471,"confidence_low":83.95054079535055,"sample_count":null,"score":87.62626262626263,"score_display":"87.6","source_stated_rank":69},"run_fingerprint":"468323c69be9c1fd783c9c68c5b62d2493666196ba42525d73b776f877e6c69e","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":321,"at":"2025-11-17T21:28:29.934Z","basis":"evaluation_started_at","evaluated_at":"2025-11-17T21:28:29.934Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ded1ea73a8e63851cfdad901fab63ff556e8120f6c5825c5127ff8520e340b03","metric_details":{"best_score_across_scorers":"0.8503787878787878","model_release_date":"2025-11-13","stderr":2.125042208224982},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"V3YQtSEnQ3TPZEPPgkCKQM","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FV3YQtSEnQ3TPZEPPgkCKQM.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/V3YQtSEnQ3TPZEPPgkCKQM.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"eda8a4a422e04ed4ea6696888b2e0103d55df73e7bde121f54c9b5be79a8b839","result":{"confidence_high":89.20296151599975,"confidence_low":80.87279605975782,"sample_count":null,"score":85.03787878787878,"score_display":"85.0","source_stated_rank":91},"run_fingerprint":"cbf48ac185891ed0a11785b23334351507658ecbe26be7aa4dfe737a69e73165","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:38:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:38:38.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5b232652ad14026f6cb09f7fa5137f284e137b9e5002868645b92c979ee2363f","metric_details":{"best_score_across_scorers":"0.6666666666666666","model_release_date":"2025-11-13","stderr":3.358618145732523},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"UT5fVL4EGvSTZBKMTPy9YN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUT5fVL4EGvSTZBKMTPy9YN.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/UT5fVL4EGvSTZBKMTPy9YN.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"a928f5e3f3693f09c7b49f4b7315f6aaecffca5fd25494894d0df927b00b7d12","result":{"confidence_high":73.2495582323024,"confidence_low":60.08377510103091,"sample_count":null,"score":66.66666666666666,"score_display":"66.7","source_stated_rank":192},"run_fingerprint":"259546b3d1d32807a9afb12a290919cac10f3e103b27fc6b4dbb6a637438c3c1","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_366cb76c94fc83c936cd8d4c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a81643784b5fbd68f33d414ac30938989741c9f35b09cc86ad7cfa440d7bbd63","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.018514,"date_is_proxy":true,"latency_seconds":23.056,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.337},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.1-2025-11-13"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d377dc667903a389772108abf03d8f1f8f4868fcc9cacc0ab26a6fe267d41cbf","result":{"confidence_high":87.03752,"confidence_low":85.71647999999999,"sample_count":null,"score":86.377,"score_display":"86.4","source_stated_rank":42},"run_fingerprint":"307c8fbef4c05e3e903da1765a782f6a5213bd5aa9bcba1b3d6a0f1896556c10","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":325,"at":"2025-11-13T19:52:15.595Z","basis":"evaluation_started_at","evaluated_at":"2025-11-13T19:52:15.595Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8e49afab56d817c1150b0a3d5a1f98994c438c841075858bf43fe36b64da4352","metric_details":{"best_score_across_scorers":"0.8861111111111111","model_release_date":"2025-11-13","stderr":4.0479727089927025},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mYKKTdV27TRJv9FpUg7rkn","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmYKKTdV27TRJv9FpUg7rkn.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mYKKTdV27TRJv9FpUg7rkn.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b2b66eb3c4c6788182a673f4582d4c4e64ce5ea5e7355e9721b1b7a059b65779","result":{"confidence_high":96.54513762073681,"confidence_low":80.67708460148542,"sample_count":45,"score":88.61111111111111,"score_display":"88.6","source_stated_rank":77},"run_fingerprint":"d7e044df9a530744e529209c1bad5a6159802006d017c40142dc5e8c93b4f303","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":321,"at":"2025-11-17T21:28:27.888Z","basis":"evaluation_started_at","evaluated_at":"2025-11-17T21:28:27.888Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c6b7fbd70fad08f755223ec924ade6b0f5dbe2eafce0594590c49332720a45aa","metric_details":{"best_score_across_scorers":"0.8555555555555555","model_release_date":"2025-11-13","stderr":4.387258359693737},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"k7inKYrGDT2LFcCiQhGfTP","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fk7inKYrGDT2LFcCiQhGfTP.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/k7inKYrGDT2LFcCiQhGfTP.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b424432bac95bed48aebcde0fb350ef96960f1a5a53880ad0ed3273a28412c3e","result":{"confidence_high":94.15458194055529,"confidence_low":76.95652917055583,"sample_count":45,"score":85.55555555555556,"score_display":"85.6","source_stated_rank":92},"run_fingerprint":"5748961c51bb75b7249e6183150c97291c2f11a3a40a94eb0d2adf694679629d","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":313,"at":"2025-11-25T15:18:46.210Z","basis":"evaluation_started_at","evaluated_at":"2025-11-25T15:18:46.210Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8fc581c4ece66fcba4bc86ca07cc8bf700d8fd111383e2a44010600d5a63b274","metric_details":{"best_score_across_scorers":"0.6388888888888888","model_release_date":"2025-11-13","stderr":5.882131152153405},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6J9aPwieMfq2eSse8ZknBW","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6J9aPwieMfq2eSse8ZknBW.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/6J9aPwieMfq2eSse8ZknBW.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e7132855da36c8c3b3e64ae06b19516ef5535f4c571de6d0584db803f8198390","result":{"confidence_high":75.41786594710956,"confidence_low":52.35991183066821,"sample_count":45,"score":63.888888888888886,"score_display":"63.9","source_stated_rank":157},"run_fingerprint":"9cc202554bca3f6cf5580883e8da2cc622e5fa0fb062456d4bf95b1a8f69dcf9","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2eaaed7b0da64d3a38c5c7e4","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:24:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:24:39.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"802d04d28b87fc2dfba0e278a97cecb6c023baccc7521cabf70f1f84582b3384","metric_details":{"best_score_across_scorers":"0.37777777777777777","model_release_date":"2025-11-13","stderr":7.309112127323451},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"H5NgPXUD57sbusTLp3yKQE","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FH5NgPXUD57sbusTLp3yKQE.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/H5NgPXUD57sbusTLp3yKQE.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"97c95cf5de9424aee8a06d8c25f26b8efd82ffde971803cfcd0952e9b611411d","result":{"confidence_high":52.10363754733174,"confidence_low":23.451918008223814,"sample_count":45,"score":37.77777777777778,"score_display":"37.8","source_stated_rank":209},"run_fingerprint":"6dc380b36ee14f185d0d1609af5bc23bc0b1106c57a265617dc6db04301704ed","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_366cb76c94fc83c936cd8d4c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fb6357b93e367b60b835b87417ac1ebac30d7f14c7690b4d284e3b408ccf62e1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":1.1704,"model_release_date":"2025-11-13T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-1-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-1-2025-11-13-thinking-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e1fad15c71d90a914eccc3e146a77b63abce2bec16f2cf125a4be0372e9d72e1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":17.64,"score_display":"18","source_stated_rank":133},"run_fingerprint":"60170d16de135de2e469e026efae34550055dce5efb7c6f827e763631a3a4059","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b69533cac462c782aebfff87478043e7d9b0224b1d1ff31efa5f1121a9df0552","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.4206,"model_release_date":"2025-11-13T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-1-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-1-2025-11-13-thinking-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8e18e67e20f0d34686b6f0db520f6f466cafff00235070a495b9b3b48a3743d5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":6.529999999999999,"score_display":"7","source_stated_rank":154},"run_fingerprint":"3ec82491c16a492a47cbb06a890e8c89f2539066006fe901572873f8e3a16eb9","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3e8040e7258f57318070ecab8c7398905d0e6d5505d556932f080075bb342373","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.129,"model_release_date":"2025-11-13T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-1-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-1-2025-11-13-thinking-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"aa665ba5b49ac0a14e9f2ee8e3b369e0c7777ce488af11c470c946c7e7bb7c76","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.94,"score_display":"2","source_stated_rank":199},"run_fingerprint":"7ffba01c09903215c5b7331f3ac62eedefd884cf2749da9d9540aeb8c738e963","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2eaaed7b0da64d3a38c5c7e4","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"094ae9c4d4f01dab88dab52a73d95597942704f27d3606317918601569cea6a7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0581,"model_release_date":"2025-11-13T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-1-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-1-2025-11-13-thinking-none"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a4927771a0a84333a66f7fafc59c49d4d1644380c19cd98088665758e22e9e50","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.42,"score_display":"0","source_stated_rank":235},"run_fingerprint":"ad2825ea67cba28f1f964833f8f0e13579e9dd587a99dc314f21ad75856e6b90","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_366cb76c94fc83c936cd8d4c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"90dfabdacae483b5430e2cfea7108532bb8207c8fce4f2283819efa9d6939017","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"1.1704","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Thinking, High)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"48c43572a61075de6f1d838784047aa1d9966c94a04a79e696b77c87ec2f2a12","result":{"confidence_high":21.913129,"confidence_low":14.050213,"sample_count":360,"score":17.64,"score_display":"17.6","source_stated_rank":130},"run_fingerprint":"2fba9fa1fdf556129a90985acec915548faf38071f8f21dfd1f46a6ca5adabd6","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5df073d26928d61fb5f890ba3f099af30213c27154d8a59841a819f5b234c5cf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.4206","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Thinking, Medium)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"385a13c51d006f256b967a66766662e1954a99f78cb737cbf282c7e8d592bd19","result":{"confidence_high":9.568723,"confidence_low":4.409228,"sample_count":360,"score":6.529999999999999,"score_display":"6.5","source_stated_rank":149},"run_fingerprint":"1f98fcc1290cf92b989fb7e170776c1aa1ace0e96d640b7986c02206997cdb19","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2b815391ebb8d75a7b8baa57655d0b8fff77f0cfe6dba97e71a5dc174f35e2ff","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.129","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Thinking, Low)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"2915fa490863616e7be791ac10ab22a1af77fa75846da31f08637be2484a0e1d","result":{"confidence_high":3.952794,"confidence_low":0.942084,"sample_count":360,"score":1.94,"score_display":"1.9","source_stated_rank":193},"run_fingerprint":"9209bd60e0049e3de561d3b7d189a7fdd178d9b17483271064a68bdfc4647ebb","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2eaaed7b0da64d3a38c5c7e4","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c2fec077407b9cdebf16f393b3895d98113463d2be9227a4754ab06d27e4563e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.0581","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Thinking, None)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"393a50ba97862dd0a2b0d7479b38cc833ed625b632b2f1238ee5fee2d092a3f9","result":{"confidence_high":1.789437,"confidence_low":0.097538,"sample_count":360,"score":0.42,"score_display":"0.4","source_stated_rank":226},"run_fingerprint":"b771fda9a9177c1eab833676f3c1feb233b323f07114475130c74c3b6c25441e","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_366cb76c94fc83c936cd8d4c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f8cddefd8a488c2ef2c843dd894372034f7fbf63cb105508fe3823cfd94970c6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.002764,"date_is_proxy":true,"latency_seconds":75.307,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.899},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.1-2025-11-13"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"37fc0b4893993458e5906406198306583f939bec383ab98047128260647326aa","result":{"confidence_high":84.94104,"confidence_low":81.41696,"sample_count":null,"score":83.179,"score_display":"83.2","source_stated_rank":33},"run_fingerprint":"ba809fc47c252463ca99e7e8019747655830f72eb7884d28f4ea47104c15328d","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bb2a9559eeef24aebc253b406c4a29b0e484ce91e2e08d5939eae5603da9fd58","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (high)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"0853f78b9e289cc0e250f444712e8dd3a8901363a4379256fd72468a457f8cae","result":{"confidence_high":12.56782,"confidence_low":1.780808,"sample_count":71,"score":4.857142857142859,"score_display":"4.9","source_stated_rank":111},"run_fingerprint":"524fcc8be039b0088726f2c9fe04ea5adf80ff203d58cd244b53287b34831a9c","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7c8f4282cd68118ad34e0908c159509487f3332e178c1639ee6133b7ce94e9c2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_high","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"2ef4f664bc752406a303de62e789284339a6581f4537d42978aa376d823bcc66","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.2,"score_display":"53.2","source_stated_rank":45},"run_fingerprint":"f147e60f810c37fafa7f4a6faa050c157a92225276947f0792effcf34e046dda","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":300,"at":"2025-12-08T14:58:42.139Z","basis":"evaluation_started_at","evaluated_at":"2025-12-08T14:58:42.139Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"9aead0d1a516136f87615605bace6f075a0a0af3f0d4f108d2e131c66e4498bf","metric_details":{"best_score_across_scorers":"0.32","model_release_date":"2025-11-13","stderr":4.688261722621506},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"8aknqG8nWu3P5CoBcetqte","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8aknqG8nWu3P5CoBcetqte.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/8aknqG8nWu3P5CoBcetqte.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"26416b1048593313cc26f35ca2161e6ac4585a2b14043586b3392b37c255ec82","result":{"confidence_high":41.188992976338156,"confidence_low":22.811007023661848,"sample_count":100,"score":32.0,"score_display":"32.0","source_stated_rank":47},"run_fingerprint":"cdb7d0573960dd7f91c619e97723bca64533900b049e149b5572c6c1b03f9e45","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:24:56.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:24:56.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8050f440a9f981f7ad3ce6c141b29f33be2d112d271a5eee1ab54d320941fe75","metric_details":{"best_score_across_scorers":"0.14","model_release_date":"2025-11-13","stderr":3.4873508801977695},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"SRH8gZhxzUjFtbe9Fo3nyg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSRH8gZhxzUjFtbe9Fo3nyg.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/SRH8gZhxzUjFtbe9Fo3nyg.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"845e1c27616bd18270118ed610301c6bab2fdc3243db68b2be3b73c911f173b7","result":{"confidence_high":20.83520772518763,"confidence_low":7.164792274812374,"sample_count":100,"score":14.000000000000002,"score_display":"14.0","source_stated_rank":113},"run_fingerprint":"d19a5fab082db909a859380135560456b7b900068f530cf981c082d62c1cf5b2","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2eaaed7b0da64d3a38c5c7e4","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:45:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:45:50.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3253ee4e51d8cdb393e27567013a1428483ee7ab0c29fc3f096ab9219c173a80","metric_details":{"best_score_across_scorers":"0.17","model_release_date":"2025-11-13","stderr":3.7752516806863685},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"MTWXxQWmaQ5kuNPo9mhoMX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMTWXxQWmaQ5kuNPo9mhoMX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/MTWXxQWmaQ5kuNPo9mhoMX.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"05cbaba56e2081679d4194e18465f6ecb011d000c3f9c84dc5ea5b19f3a103a9","result":{"confidence_high":24.39949329414528,"confidence_low":9.600506705854718,"sample_count":100,"score":17.0,"score_display":"17.0","source_stated_rank":104},"run_fingerprint":"04d65e212e3f3ccb25da9fe6cc846ffa0db8d23e90735342a8b39a09e1eb02f0","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_366cb76c94fc83c936cd8d4c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"104db3fbbcd96183f2f55038ff9d06c890f943027f6a426138138286b465d94a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Thinking, High)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"eb04a4be031c049fb589d6777a50cf93f1483a73693dc20532c216d64482e27e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.83,"score_display":"72.8","source_stated_rank":129},"run_fingerprint":"0bc2cde40b1d7eebb76639823337e9235815726994e47a869143f3a8318e6157","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9a0bf75b68bba1d8c9f5b8dbbe663bd8e38dc44125ce6ca410a4cd9ca134a3d6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.674","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (High)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"0776ab36ea6b6acc01cbe5807fa40cad10cc8ddd4c62d9de4b4f1cd36ad95397","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.8,"score_display":"72.8","source_stated_rank":130},"run_fingerprint":"848fb500324f408032f3235fc70761f34658f02b66b69eb844c7e2af4d00f095","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b36ee6f6dd837ae8bd30ad864359e0763d416df5636de08669b17e00caddb2f6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.3149","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Medium)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"7f95d3f596e8a77688503dcdb515c0a11968e3fc46fbbe2fafa444064c64713c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.699999999999996,"score_display":"57.7","source_stated_rank":154},"run_fingerprint":"9c4959f8ca7a2481d9bd8380539ef737756c83065e5ba675ab436f66372c08d3","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c09eadb364b48d241bbd901649c44d315dfec100fbd0c157d3df70cc677a2c2a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Thinking, Medium)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"dabb1eb106964086ad69af32961a530b2dbc243d6e9282e50942c51e4dd309d7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.67,"score_display":"57.7","source_stated_rank":155},"run_fingerprint":"bfd8270cc83853ec12f306d210c167f3b534443fde7a13e8cb74f4e0dff6c878","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aafbb9fb42a3df95ba648ec4295277da63df046331ffd2433ebe3f241f1afe45","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.1","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Thinking, Low)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"7a6a00542bc8970309fd5bbf3dc2a69e7c1d338093ef555b25652ded15f96500","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.17,"score_display":"33.2","source_stated_rank":195},"run_fingerprint":"ee8c67293e5b9af0a25062469c7208b9b45f3c285d5e9b355b1e6b583bfe0808","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2eaaed7b0da64d3a38c5c7e4","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"41a8b912cc5c5d9fe8305c4d6f35a106aa3b941610758423f8c56ba9d98a730b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Low)","source_effort":null,"source_model_version":"","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d2248613ca26a4491ac98de9199e82e91012e9beb8d54aa2f8c3fd1b2bf70a92","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.1,"score_display":"33.1","source_stated_rank":196},"run_fingerprint":"0bd0c907b3bed22ecb6805adac1c2cdcbbec91511185b3825587a18ca8369ba1","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2eaaed7b0da64d3a38c5c7e4","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"34a8befe294d765f2038d71b2d1b102818562ca9f1b4d37af5a7f32ad8964aa2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.0338","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":""},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (Thinking, None)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a8c0c6786d45e294428aada576e2141792370f4c2697d1834fa2f4657deadc1d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5.83,"score_display":"5.8","source_stated_rank":256},"run_fingerprint":"ca16a5ddbc2c5c41f5896d22a659987ae59275872f385f901651f31f41d3ca38","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_366cb76c94fc83c936cd8d4c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"86146aaa286fa98814db2b78f7504e07f5f09d5bdbbab8f0a6cc5540251bb7f6","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.713431,"date_is_proxy":true,"latency_seconds":500.651,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.055},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.1-2025-11-13"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"7f77adf9c88c774711296ae34f4392aa307260fa4e1f0d1ab224ed160efb4519","result":{"confidence_high":73.8278,"confidence_low":65.7722,"sample_count":500,"score":69.8,"score_display":"69.8","source_stated_rank":64},"run_fingerprint":"b8f00849aad5698221c69a3f909c3f9f87a25d0a8fb9dad907112fcf50886038","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":229,"at":"2026-02-18T12:36:35.180Z","basis":"evaluation_started_at","evaluated_at":"2026-02-18T12:36:35.180Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e641fb74b4ff25ceb0b0d29fa043d5f651af2797b0ac051e7b645af08cd867bb","metric_details":{"best_score_across_scorers":"0.6797520661157025","model_release_date":"2025-11-13","stderr":2.122974956319103},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"YoHtZpMNdsRE6qdj2K4Ucj","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYoHtZpMNdsRE6qdj2K4Ucj.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/YoHtZpMNdsRE6qdj2K4Ucj.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"53db8892120e3ab02c09ce7aaee7b3302a9afcf76b80ff794ede3d96622f50b8","result":{"confidence_high":72.13623752595569,"confidence_low":63.8141756971848,"sample_count":484,"score":67.97520661157024,"score_display":"68.0","source_stated_rank":27},"run_fingerprint":"09a4af7f9145a1723d281bb4095eb9914f5d48a06f9f2256d83c992784b04a74","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":319,"at":"2025-11-20","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-20","status":"stale"},"measurement_id":"f31c0f49b598c9fb300247b421ff3ba39c178b01966a04bf83c7f3d1cf3cb2d1","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2025-11-20","folder":"20251120_mini-v1.15.0_gpt-5.1-2025-11-13","logs":"s3://swe-bench-submissions/bash-only/20251120_mini-v1.15.0_gpt-5.1-2025-11-13/logs","model_display":"GPT 5.1","model_org":"OpenAI","reasoning_effort":"medium","resolved":66.0,"site":"https://platform.openai.com/docs/models/gpt-5","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: gpt-5.1-2025-11-13","Org: OpenAI","System: Attempts - 1","Mini: 1.15.0"],"trajs":"s3://swe-bench-submissions/bash-only/20251120_mini-v1.15.0_gpt-5.1-2025-11-13/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"2a766d0f4c07aff44b149a99fb5bca9eb813927b26abaa8b44b46aec2d960eee","result":{"confidence_high":70.016185,"confidence_low":61.739827,"sample_count":500,"score":66.0,"score_display":"66.0","source_stated_rank":40},"run_fingerprint":"41a86ee4273fdf779a980c0532beffb19866d3164c9d395d27b244598c5c6fc7","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:04Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"26ff7a91be3fa723f4378b237f36509e4ba896056745c2b4c38d9f44d8819b17","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"GSO Leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_high","source_row_date":null,"source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"01cb454390bd6d82cb9a851b14464ba4a4864e6d59cf0d74408c793b8ea05cf4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.700000000000001,"score_display":"13.7","source_stated_rank":19},"run_fingerprint":"420c1598ff4c919e5e851e53ccf41e335241d3cf9651cb51500ea8fc50833cd8","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":291,"at":"2025-12-18","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-18","status":"stale"},"measurement_id":"48941f54e4eff237de6759b5515ea2d4a334523728b2f4474ca62661423a8175","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_unknown","source_row_date":"2025-12-18","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"deddfb95c4c90b88f8568e14b13102cf0212f4a610fc080c85e0d6aba217148a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.73,"score_display":"13.7","source_stated_rank":18},"run_fingerprint":"9e405d5b701ff64e59f34b10c1fed96988aaecd69e4a2054055194c50cab8b98","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3a0ef7e05965505ef76146c0","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c8afd903d4e07715abfaaede0ffe7efb78a43ffbfeccd9ebe4ac652273afc07c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.125706,"date_is_proxy":true,"latency_seconds":141.24,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.983},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.1-2025-11-13"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"51de70a760b34d6ae4d382d903380c1935be6d419e168d64da82ba172d9cf343","result":{"confidence_high":88.41268000000001,"confidence_low":84.55932,"sample_count":null,"score":86.486,"score_display":"86.5","source_stated_rank":22},"run_fingerprint":"510ceb8186e67e85e829913e1a8c4887f34ad2cdd5f3009884f2ad6804e3b8d0","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"03bae2693d41deb7cc42fc9f34bab4b2439042821c30ed7431aaad7cfb5267d6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1 (high)","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b1c2fa312286f3d10996ed85c71f12bd264c20e0b9ccbc43fe486fd6e266e88b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.287037037037,"score_display":"43.3","source_stated_rank":130},"run_fingerprint":"891100364bf5f1484ac2c837dfb96f1ab29595eedbbbd29214b832e00748882c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dc637da8fc04893efdc76df7e78687a43553db162cf63052e46be05219e741af","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.315236","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.1-thinking","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"a6d89d9cdc6b737bffc5b0b25b7cef378a9abd72faf61acd5608974604fa4ad6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1192.15,"score_display":"1192.15","source_stated_rank":35},"run_fingerprint":"58d8553693bb7feb2064512e836a3bba6635b30414ebe8616cecb6d5b559bf2c","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"62f7bd02e03177998a12febd0107f3a451aec607e81a029ab311e947698df488","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.574926,"date_is_proxy":true,"latency_seconds":1836.188,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.254},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.1-2025-11-13"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"2abdf6dee85c1ca9af55513d664f73a54c6e665a7b42d018d883c36e104b5798","result":{"confidence_high":32.94384,"confidence_low":16.26816,"sample_count":null,"score":24.606,"score_display":"24.6","source_stated_rank":70},"run_fingerprint":"4930f052ecfc478cc626dce7b4ac56efe892651a07fc08c925911fc36fe5aaf5","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bb64b5c720d1593ae9160adea05cc46202d445632ab807542bc6ece98e75f380","metric_details":{"license":"Proprietary","variance":34.28539281766522},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.1-medium","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c14b79818f1235ea38d42156e66ecf05bae097b509a6b021dd3355cff537d307","result":{"confidence_high":1406.266382846653,"confidence_low":1383.3137425052557,"sample_count":4911,"score":1394.7900626759542,"score_display":"1395","source_stated_rank":90},"run_fingerprint":"ed8549f77031753728d16c453c81827ac11e359a94b871eb41ac75815c2a3207","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"92315fadfd7a7b82e23a8d8982355285e954bb78733a1b7086364d87ffe7f336","metric_details":{"license":"Proprietary","variance":18.576994174052853},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.1","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d13197a9583eaca1d9e00f494183b2fc32eb11dced8a709c10365ee70f157ffd","result":{"confidence_high":1349.4930243182346,"confidence_low":1332.5977281525088,"sample_count":10718,"score":1341.0453762353716,"score_display":"1341","source_stated_rank":109},"run_fingerprint":"35b5071821975de46014e0193c621e129bdf777f95a00d1e56c8730aa5e35539","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b925b2f1d145538058961225","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"3274d290610fac4a46f347bba98758b59eb8d1e1b7507d10961c2e7f51fafb41","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_medium","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"00fc0d532b9aa3bf6c2c57f1a49604f34ed542341bbd07c52e91446cb3984def","result":{"confidence_high":1403.69,"confidence_low":1380.37,"sample_count":4751,"score":1392.03,"score_display":"1392.03","source_stated_rank":87},"run_fingerprint":"f1c26661d2b0e40a8e6a5ac4849d2168148625bce0a94ace484b9e1bd726e913","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"5658e974c25d8e586dc0cd7d0af15e0094a0463f8d160bfd4c374e77b5a6cdbb","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_low","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"9f7ffebd2d2e975f7df748bc3c687bf1e3cc8a4f224c025e2099990fb4c57145","result":{"confidence_high":null,"confidence_low":null,"sample_count":6540,"score":1359.0,"score_display":"1359.00","source_stated_rank":101},"run_fingerprint":"8992707a69c9367b1d14347e0ac00d9316f1fc3ec1a28f34dca37daf4322ad8d","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2eaaed7b0da64d3a38c5c7e4","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3348879c45d3b161998f813ea649c5366ccc08d24447e0ff2cd304468ff677c4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"467eda6f51cad1529f0344f9f15a4cb8900a9d77786d4aa2192672825fdd447e","result":{"confidence_high":1349.46,"confidence_low":1332.34,"sample_count":10326,"score":1340.9,"score_display":"1340.90","source_stated_rank":106},"run_fingerprint":"378a692cb6ddd1da6799d8ab5aacf48cfeb51a3a0dda46c5067261e12f9d0010","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_27a9b3cb634fb345ac8209f0","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9a08334f69532fba1fe1dc7b2fc4dffee15e74f4bda3ce0382ab1733988a677b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"b6a4d86486624be5cca6ad8599ec08e1124238eb955a20cfbad728c24bd3796c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.77,"score_display":"60.8","source_stated_rank":48},"run_fingerprint":"9cf999a7e83699f87d5cd4cb01117d57e4dbd34475afb8f8b6e7ba337368e8fe","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":323,"at":"2025-11-16","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-16","status":"stale"},"measurement_id":"7d098c81838bc6dd5735644b831fc80931c034c328f480975e90a41b3bd82a5f","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":"Assuming medium based on GPT-5 being run at medium.","standard_error_points":2.8000000000000003,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_medium","source_row_date":"2025-11-16","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"c777085829c7f42db5e576dcbdb93e5bd801cf3889ffad258d6e93f1e4395ed5","result":{"confidence_high":53.087999999999994,"confidence_low":42.111999999999995,"sample_count":89,"score":47.599999999999994,"score_display":"47.6","source_stated_rank":75},"run_fingerprint":"a7f45eeda480f1cdc1b4ba5931d08d4b263949e397a8cc0590ef7b84e1329780","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_01f55deedf8de2a463a745ff","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":327,"at":"2025-11-12","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-12","status":"stale"},"measurement_id":"f8706aff8bd42f21106b383c92fd3b0970bfa70276c0776a57c23424b6ebc922","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"standard_error_points":2.7616039281,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GPT-5.1","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_unknown","source_row_date":"2025-11-12","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"72c8eacbf62f1e8df26a0fa8c9ad9710ea617fff6bcc261f5f93c05e19baa3f8","result":{"confidence_high":52.997013361976,"confidence_low":42.171525963823996,"sample_count":89,"score":47.5842696629,"score_display":"47.6","source_stated_rank":77},"run_fingerprint":"a6651397f42a0591579f701e3cbb1b8c75ec8d5818c710bd36a5f51402fd0f20","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9acc12025fb924035a896928","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"7dd1cd51b42acc9d37991377a77b5a15af66038cfca895408ce80c7b8f1c7dea","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.315","mean_standard_error":"2.7","model_release_date":"2025-11-13","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.6,"upstream_source_url":null},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5.1","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"bf00e4578884c600eab5e378a3e1ec6ae56feba5896e0aac921f50d172a16b82","result":{"confidence_high":22.596,"confidence_low":12.404,"sample_count":480,"score":17.5,"score_display":"17.5","source_stated_rank":41},"run_fingerprint":"30295313368706c4ac8ff876cd209bae92ad54f702dc264285624dbc28f848fa","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"bec23f73798ffe1d695fc665785d4b817f5de72f33eef598474cfc0bf7bf33ca","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.315","mean_standard_error":"2.7","model_release_date":"2025-11-13","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.6,"upstream_source_url":null},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"daa0e0940e6327a2c6ba5afabb9e814585708314acc41603b8e223a58732cf18","result":{"confidence_high":22.596,"confidence_low":12.404,"sample_count":480,"score":17.5,"score_display":"17.5","source_stated_rank":41},"run_fingerprint":"21b14299fc7eb4781a16fae2fcf82e9599e2cd19087356c1b4279ad51fd4f6ef","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3825115dce6b84cf3d3ffa0a","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"95bd65841752d5ed0e2aca7f2c7be75b8922819b4ee2d386e7439fcf4f94de1b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-13","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.1","source_effort":null,"source_model_version":"gpt-5.1-2025-11-13_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"b2fc64da721ffa54d136f2e30205fd93a251d487411373d4ee28a27083d42d9d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1473.432,"score_display":"1473.43","source_stated_rank":47},"run_fingerprint":"9a4053facce1b047f1e6386bf1665975b9309e02aefd93a70764e93cf2e1fd2b","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cd54c0156f4acd0d4d181e31","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"706b960b6db47935d7e2730a1e47ecf891463fe535ed152334a10eb022822706","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":272000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2025-11-13","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · GPT-5.1 (High) · gdpval"},"model":{"id":"openai/gpt-5-1","name":"GPT 5.1","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-1","upstream_model_label":"GPT-5.1 (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"ce0a4feded3127bbb7677d77771af4373586754e4fbba0e9aa101ea4ac4c64a8","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":812.62,"score_display":"813","source_stated_rank":null},"run_fingerprint":"814c6acb692a61c85a6b7c2906dfb56f085401bba9aad4974421aac1458f25a0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.1 (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_30f1bfb383c02c782db74a79","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5251c88efd5e78cbd2f4ec7f4bb465eef30744bd9438d090e602d0d9c07dcfdd","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-08-14"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen 3.8 27B; model release: 2026-08-14","source_accessibility":"Open weights (unrestricted)","source_model_name":"Qwen 3.8 27B","source_model_release_date":"2026-08-14","source_model_versions":["qwen3.8-27b","qwen3.8-27b_xhigh"],"source_reasoning_efforts":["xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"acbadb8716383f08981445e5cd05a9cee4a51e84bf99254f703538e73e430e70","result":{"confidence_high":151.63,"confidence_low":147.45,"sample_count":null,"score":149.41,"score_display":"149.41","source_stated_rank":54},"run_fingerprint":"ea764146e8001c766e78cb50b8aab279956f88ac2488476ed471fd8d446c1d60","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e182cbfed410a4de7a2d42dc","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"33edc43907ce6d4120b0c67fdd1c32db7f2f6b272163ad6925952f238bb78a4c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b","upstream_model_label":"Qwen3.8 27B (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"8de9d8f47604e1e5286abcf72822afbea00b9b306ed43c3a809c85337ae1a5f6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.696286,"score_display":"33.7","source_stated_rank":null},"run_fingerprint":"b7a5752724136236ff0479b388b056b551621e8c26984f178023cdc01f8c2ae2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"861446c8a2c781f7416218cc561c875666c1084fa149cc5b4b83e00052d25593","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Medium) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-medium","upstream_model_label":"Qwen3.8 27B (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"ffeaa0ac0bac20697366db8554ab65cf9bda91854da7b5e6fa49bd76531cbf45","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.550823,"score_display":"27.6","source_stated_rank":null},"run_fingerprint":"62587430c5ca6f33a7071fe56d48560fc2cde6a7f59399ee28eea932735c0a8d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7cf5024058a8f6bac96455431e20ab74c7f22602958d4a6b2c93581feb186add","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Low) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-low","upstream_model_label":"Qwen3.8 27B (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"aa9920458544d051bf8007e5134439e25c73a6a2870d2fca5555b64428931489","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":26.204798,"score_display":"26.2","source_stated_rank":null},"run_fingerprint":"3cd4eb3e5c3638a738d0cf85ceccc29c7d86ab8d6b611b6c4ac404df32d19234","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1e2b2109d1f411f8edfc4fda2ad78f864a5e988b8ca6d31a1233b07411c8ddc5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-non-reasoning","upstream_model_label":"Qwen3.8 27B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"fd61fc362ed47c40eec64aa3bbff116e8b850be764f1c52b3b02ce2493372337","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":20.150243,"score_display":"20.2","source_stated_rank":null},"run_fingerprint":"74eb3ca1022b16d22490d95122200d5dbb788e2639898b04a181104e096c6b46","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6be1f89a55fb013cd65f763a9c109377af9c4a0040a0b71677dc5febc6c2dc24","metric_details":{"license":"Apache 2.0","variance":7.633943015775435},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1443.1150267596286,"confidence_low":1432.2844260866045,"sample_count":18462,"score":1437.6997264231165,"score_display":"1438","source_stated_rank":99},"run_fingerprint":"f937955eaac0baffd8fc215c7198671969fdd363cc5bd2b44564bfdb63c46fb9","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c384460647f97ae85afeb51c","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4998439e6d0747705899d1b6f7e869f434deb7c77d35a11ab62510e144b17315","metric_details":{"category_scores":{"Agentic Coding":61.36366666666667,"Coding":75.689,"Data Analysis":76.58566666666667,"IF":72.65825000000001,"Language":74.34566666666667,"Mathematics":86.21275,"Reasoning":80.02875}},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.26910714285714,"score_display":"75.27","source_stated_rank":38},"run_fingerprint":"4a1dfec777284266e0bb6688b6c121d00a692ed949f23606d1fe37dd5d93790a","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_da362f90a33ff4c7924ed94b","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_287aa4bcb31b5b1ba26e562a","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"72d839b6fa6a023ae351cb1baf4c4f51eacaddca4530e13b72c87e577729ad9e","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Text Performance table; General; Multidisciplinary reasoning HLE; Qwen3.8-27B column; HLE methodology note"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":"GPT-4o","run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE evaluation judged by GPT-4o; task count, split, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"c93a278e8ec3e88d3cf8590bc92e78109edbf3fa8907cc99906d802003896656","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.8,"score_display":"30.8","source_stated_rank":null},"run_fingerprint":"382269a32ca84b946eb107db590b7b7c69df58670099d25d7d45c8bdf4a6fc21","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Text Performance table; General; Multidisciplinary reasoning HLE; Qwen3.8-27B column; HLE methodology note","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f97c4e36649aacbbeb6c72f4","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"evaluation_at","evaluated_at":"2026-08-14","first_observed_at":"2026-08-30T10:48:10Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ea672ed1384a1a3fba5963b2dd626fd63e44a7d3130cd3cf94c689a95b12e976","metric_details":{"modality_lane":"not reported","notes":"Judged by GPT-4o.","num_parameters":27781427952,"pull_request":22,"result_file_url":"https://huggingface.co/Qwen/Qwen3.8-27B/resolve/refs%2Fpr%2F22/.eval_results/Qwen3.8-27B.yaml","source":{"isExternal":false,"name":"Qwen3.8-27B model card","url":"https://huggingface.co/Qwen/Qwen3.8-27B"}},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":"GPT-4o","run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e8b13bf3ab8af909356ebd855caff92593b1823033b1f522ec18c0e5a218c495","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.8,"score_display":"30.8","source_stated_rank":40},"run_fingerprint":"f847e40c4d3aaf90f6b2d169282b40835ee96a98356435b7fe07d24eaae9c53a","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_77617168b9b81695c0be1207","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7c4a3f5730aa462b699cbb5d064f7916e3f1ddf72775b3a60eaf1efc16d3b6da","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · hle"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b","upstream_model_label":"Qwen3.8 27B (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"80a0d85a4e906cd1bb8506e00f47baa85eab76e892cf7d707f3e38a92518da1a","result":{"confidence_high":35.944834,"confidence_low":31.952908,"sample_count":2158,"score":33.920297,"score_display":"33.9","source_stated_rank":null},"run_fingerprint":"ee0e99355b03d5eac6c68e6f69d25029193d31d89925e72f5f19e910e14a7b7a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e765805e0b8efd8723b85a58a4003e34b2be39c4a9037c31d259ca064a0ebf0f","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Medium) · hle"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-medium","upstream_model_label":"Qwen3.8 27B (Medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"80a0b8f837f8736a5c4b2fa095adc419f3afc600022382a0d5aa06b056fced95","result":{"confidence_high":15.618831,"confidence_low":12.68304,"sample_count":2158,"score":14.087118,"score_display":"14.1","source_stated_rank":null},"run_fingerprint":"84e506fe9cfb28d6b8d942571d67a69172060d6370ba03b234250f2f4d1dda1c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ecee1f6a77e2e0a890ff51736ae8b26bda875775443b761212203ae2bf5a6c5a","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Low) · hle"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-low","upstream_model_label":"Qwen3.8 27B (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c6a3c4a24ad78d102d9936fa5b92209ff6de0ea874abcca6b7c590af0fe05c96","result":{"confidence_high":15.57056,"confidence_low":12.638796,"sample_count":2158,"score":14.040778,"score_display":"14.0","source_stated_rank":null},"run_fingerprint":"9995380dbf5a97fa7dfe569dd91dc55543d6e9132343e8481c46140cdb99160d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cab046071859259da79352d2f5c512862c3ca56252f7e0595e9e9bdafaa08de3","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · hle"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-non-reasoning","upstream_model_label":"Qwen3.8 27B (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"13711601e18fe30af6b50067078cd105d41a33262d56dc934139d7b51428649e","result":{"confidence_high":13.586563,"confidence_low":10.829731,"sample_count":2158,"score":12.140871,"score_display":"12.1","source_stated_rank":null},"run_fingerprint":"ecf1527ffabcf11f2aa641bd06ae63aa882c6791d6925bb8b7d42a5542df2432","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"520095dc71b76bcc808013615fd0cb017f165c2b6ec32f562709b674bab56931","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.046624,"date_is_proxy":true,"latency_seconds":116.361,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.579},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":88.889,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"xhigh","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.8-27b","zero_shot_accuracy":88.889},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"66da279e670e3f5494e2bc4ad90a03762b304678831b3c0eff2ddbc211dc5e35","result":{"confidence_high":91.98384,"confidence_low":85.79415999999999,"sample_count":396,"score":88.889,"score_display":"88.89","source_stated_rank":32},"run_fingerprint":"c448fd49be828b8e2565d3e1e2ac720a531f21cc8a3f2218d9c267d0b678332e","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2e458477deb18a172b5d4b27b973bf2ae0f1f996ffd5b12f58b26227868cfae7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · gpqa"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b","upstream_model_label":"Qwen3.8 27B (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"08c6ed54790377357ebce38df4d7eb8d1f13802885afc0aceb52cadbfff83dd9","result":{"confidence_high":93.851159,"confidence_low":85.617098,"sample_count":198,"score":90.505051,"score_display":"90.5","source_stated_rank":null},"run_fingerprint":"c57aa6a753504ed2530a8cc21fa50c51eec529d04b04d20dc179c9f3a53da5ac","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cb977a170e55ca789338091fe24938774a2b6782fe8b170201ae2f9b155bc13b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Medium) · gpqa"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-medium","upstream_model_label":"Qwen3.8 27B (Medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"226b58c9a3bbb1ce11309aa3e9cc280bede263ee2a232b930c6a9e33c68bcd17","result":{"confidence_high":88.917941,"confidence_low":78.857979,"sample_count":198,"score":84.545455,"score_display":"84.5","source_stated_rank":null},"run_fingerprint":"51906d848354f1493ffcad0e5364ac088f4fbef9b809e9cb092b187d8a689f55","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6430e78297cbf4611292ebeac97e39168f696a4514322f383fcfb7e37027aa34","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Low) · gpqa"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-low","upstream_model_label":"Qwen3.8 27B (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"04ae3836ad74baaee58d4901a91715309f8ff789ad31ba8c13c1f050cca12caf","result":{"confidence_high":88.917941,"confidence_low":78.857979,"sample_count":198,"score":84.545455,"score_display":"84.5","source_stated_rank":null},"run_fingerprint":"08b43179cf9b06ddc12636a085b879c508c7673ede368589c8aecafce74a118f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"746334a33e292d3f732f8ae74f6d4117ecdf99d1f88ab860f755133d10c4e47a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-non-reasoning","upstream_model_label":"Qwen3.8 27B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8733aa9d518c24f2601206877060390b4fb17aca3f8b8de01fc2a809c9b63b5f","result":{"confidence_high":86.567958,"confidence_low":75.857232,"sample_count":198,"score":81.818182,"score_display":"81.8","source_stated_rank":null},"run_fingerprint":"51b042464a4bace6de968862e42e4ee3b32accc2d316f173bb9a8025cfdf7bd1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_673a491f34d6df5a4774eaf2","split_or_window":"escha-gpqa-diamond-198","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-09-02","status":"fresh"},"measurement_id":"84fb4e35d934ae4ac3c22bbe007b0dc1273530bcb37d36259f903fd63e1aa359","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2/resolve/561dae0aeea1efbdafe122f2e385fdeaa99127e7/README.md","evidence_sha256":"399a0d0791ac1bfec5a2045df28d2b200a2661c0a81f2867a17dc95cd89d4986","kind":"reviewed-static-report","published_at":"2026-09-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2","version_id":"2d9354e5d0ffc5d484d1a8f95b09f83b6f4e55e4ec0903e813b0bb08f36992cc"},"source_locator":"Quality vs FP8 table; Graduate science; FP8 column"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Escha Labs","evidence_verified":true,"freshness_proxy":true,"protocol":"Same SGLang fork; xhigh thinking, 28672 reasoning + 4096 final token allowance, temperature 1.0, top_p 0.95, top_k 20, seed 42. All prompts in denominator. FP8 and W2 use different GPUs and engine dtype. Quantizer self-report; different GPUs and engine dtypes. W2 includes learned quantization adaptation and is text-only. LiveCodeBench is only the 182-task 2025 subset. Small gaps do not prove a quality difference.","quantization":{"backend":"Escha SGLang fork","base_checkpoint":"Qwen/Qwen3.8-27B","hardware":"FP8: different GPU, not specified; W2: RTX 4090","id":"fp8","label":"FP8","limitations":"Quantizer self-report; different GPUs and engine dtypes. W2 includes learned quantization adaptation and is text-only. LiveCodeBench is only the 182-task 2025 subset. Small gaps do not prove a quality difference.","reference":true,"study_id":"escha-qwen38-quant","study_label":"Escha Labs · Qwen3.8-27B"},"question_count":198,"reasoning_effort":"xhigh","result_published_at":"2026-09-02","source_content_hash_method":"raw_bytes","source_content_sha256":"3e8057f24bfaa4a5f96faca89a19de8adc8df043dc338517dda092c8f432634b","source_published_at":"2026-09-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"df2a7ba7e09abfc4c941df041dd41715fa865aff06aab262b49d83160dbdfd84","result":{"confidence_high":92.547998,"confidence_low":83.751635,"sample_count":198,"score":88.89,"score_display":"88.89","source_stated_rank":null},"run_fingerprint":"0adb66c94b851cf31f5a1d7e21219ca54bfc75d922b0e25916f78b5bebd9a651","source":{"content_hash":"b8398566036ec78cf366b63879678897049710b9ac91d17bcad9fc24f9481c9c","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2","id":"escha-qwen38-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Quality vs FP8 table; Graduate science; FP8 column","name":"Escha Labs Qwen3.8-27B FP8 and Escha-W2 study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_673a491f34d6df5a4774eaf2","split_or_window":"escha-gpqa-diamond-198","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-09-02","status":"fresh"},"measurement_id":"789cc3f5a2d1e15da34d0afa7902dea4f2926a3aa0537e4f01789b7928ecaa4a","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2/resolve/561dae0aeea1efbdafe122f2e385fdeaa99127e7/README.md","evidence_sha256":"399a0d0791ac1bfec5a2045df28d2b200a2661c0a81f2867a17dc95cd89d4986","kind":"reviewed-static-report","published_at":"2026-09-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2","version_id":"2d9354e5d0ffc5d484d1a8f95b09f83b6f4e55e4ec0903e813b0bb08f36992cc"},"source_locator":"Quality vs FP8 table; Graduate science; Escha-W2 column"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Escha Labs","evidence_verified":true,"freshness_proxy":true,"protocol":"Same SGLang fork; xhigh thinking, 28672 reasoning + 4096 final token allowance, temperature 1.0, top_p 0.95, top_k 20, seed 42. All prompts in denominator. FP8 and W2 use different GPUs and engine dtype. Quantizer self-report; different GPUs and engine dtypes. W2 includes learned quantization adaptation and is text-only. LiveCodeBench is only the 182-task 2025 subset. Small gaps do not prove a quality difference.","quantization":{"backend":"Escha SGLang fork","base_checkpoint":"Qwen/Qwen3.8-27B","hardware":"RTX 4090","id":"escha-w2","label":"Escha-W2","limitations":"Quantizer self-report; different GPUs and engine dtypes. W2 includes learned quantization adaptation and is text-only. LiveCodeBench is only the 182-task 2025 subset. Small gaps do not prove a quality difference.","reference":false,"study_id":"escha-qwen38-quant","study_label":"Escha Labs · Qwen3.8-27B","weights_gb":10.15},"question_count":198,"reasoning_effort":"xhigh","result_published_at":"2026-09-02","source_content_hash_method":"raw_bytes","source_content_sha256":"3e8057f24bfaa4a5f96faca89a19de8adc8df043dc338517dda092c8f432634b","source_published_at":"2026-09-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9a06efed1f2ae13f65cc74dd01c22b6e0b6960f93d7b9e11f89baeef27309165","result":{"confidence_high":92.130568,"confidence_low":83.168478,"sample_count":198,"score":88.38,"score_display":"88.38","source_stated_rank":null},"run_fingerprint":"7ad28f7e384997bab001938f6e954a0e2d09fa4da1d2ab45a774ca91d2e8bf54","source":{"content_hash":"b8398566036ec78cf366b63879678897049710b9ac91d17bcad9fc24f9481c9c","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2","id":"escha-qwen38-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Quality vs FP8 table; Graduate science; Escha-W2 column","name":"Escha Labs Qwen3.8-27B FP8 and Escha-W2 study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_beac9b3474dc330fed0ebc77","split_or_window":"diamond-vendor-protocol-undisclosed","unit":"percent","version":"GPQA Diamond · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"ee8bdcc905767914e3c0e429678d589a00d09947ac3186bda9523a910889cdc7","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Language benchmark table; General; Scientific reasoning GPQA Diamond; Qwen3.8-27B column"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card GPQA Diamond evaluation; prompt, shots, tool policy, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"37666acee64e42521c5a0c92defcae7e53f85111999f3bcbf4a65bcfcf864ffb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":89.2,"score_display":"89.2","source_stated_rank":null},"run_fingerprint":"d2d1ae1638bc0961a18116be407b8f18a04af9700fa6041813ed83e31daaa099","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Language benchmark table; General; Scientific reasoning GPQA Diamond; Qwen3.8-27B column","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f97c4e36649aacbbeb6c72f4","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e13113dcb60bc7dd3ade27e4d6cb1f9abb4d9d5612a29d15b369cccd68eaf604","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.010543,"date_is_proxy":true,"latency_seconds":69.848,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.356},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65536,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.8-27b"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"88d4fce71378e3b220993e1d8c44d61b1930da83419fb0a1ebe78cf93e9b4196","result":{"confidence_high":85.03476,"confidence_low":83.63924,"sample_count":null,"score":84.337,"score_display":"84.3","source_stated_rank":65},"run_fingerprint":"2fee0c1aaa6ce8549238ec17b1cc515938f733a6bd3906d2658dc12d3b7b1a3f","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-01T20:29:26Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T20:29:26Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"eacd8239c201adf6b693afec54edb3eae79d50a386a4a1d89a272b95d4f9529d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.4469396510416667,"model_release_date":"2026-08-14T00:00:00.000Z","results_url":"https://arcprize.org/results/alibaba-qwen3-8-27b"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"alibaba-qwen3-8-27b","model_type":"CoT","upstream_model_id":"alibaba-qwen3-8-27b-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3a51d79cb6ec2f0dad6101a036a1aa9a9031c99698e9e69ec67d4b2b40a089b3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.36111111111111,"score_display":"42","source_stated_rank":103},"run_fingerprint":"5f0d4d80b93ef99981813ffd874866c453bdf417aa5eefb9fb7f28be662c154e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-01T20:29:26Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T20:29:26Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c055d6dd8e4a089be9198102ba44bafa962aae1a04f775b3524d4bf55553a0a6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.2862669347916667,"model_release_date":"2026-08-14T00:00:00.000Z","results_url":"https://arcprize.org/results/alibaba-qwen3-8-27b"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"alibaba-qwen3-8-27b","model_type":"CoT","upstream_model_id":"alibaba-qwen3-8-27b-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8a43fc2e6317ca9718104be689bcec30a295d4f9f383552a3adfe833e31f2b99","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.194444444444445,"score_display":"13","source_stated_rank":138},"run_fingerprint":"64cd9c1934340d15b4dc0a8ba99f798cbeef6ad30e520176b348a8a9cca6896e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-01T20:29:26Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T20:29:26Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"35d2a92089d0af23c65514b5c7145d95ea464476476303b4392b554412c10136","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.2380080077083332,"model_release_date":"2026-08-14T00:00:00.000Z","results_url":"https://arcprize.org/results/alibaba-qwen3-8-27b"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"alibaba-qwen3-8-27b","model_type":"CoT","upstream_model_id":"alibaba-qwen3-8-27b-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c19cbd6c1e3b3bdecf865c6fc2ac421bacb447dbd73b96ebd6deecf53f9efa31","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":22.77777777777778,"score_display":"23","source_stated_rank":127},"run_fingerprint":"6044585e36980e5e8d41d45e806adf4328c2fed5ec456f147fe0dbde801d7e6d","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-01T20:29:26Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T20:29:26Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1ce029cbee9eeade4256afc9c06374cf4be2c8f63e2bac789ef45a57ec7ce627","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.05891859291666669,"model_release_date":"2026-08-14T00:00:00.000Z","results_url":"https://arcprize.org/results/alibaba-qwen3-8-27b"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"alibaba-qwen3-8-27b","model_type":"CoT","upstream_model_id":"alibaba-qwen3-8-27b-none"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"87820cf8603efc880dc2366be94c5686780fb082cf136cbed12b8b30a666eb06","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.5277777777777777,"score_display":"2","source_stated_rank":210},"run_fingerprint":"b7d3c6b04dc721c17e43f9b33aec3d3c9c14dd06c6b91bc0f3fa80e574773a0f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6389ca87f16afe2e970a92b270042ba8be0133d6c31a62d2ded9a605daaf8b69","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.012642,"date_is_proxy":true,"latency_seconds":43.483,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.883},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65536,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.8-27b"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f29831fd444a8dd05f9daa169d24d67fa172ff0ccd74b383e721e60cd60d0e16","result":{"confidence_high":85.66168,"confidence_low":82.20031999999999,"sample_count":null,"score":83.931,"score_display":"83.9","source_stated_rank":28},"run_fingerprint":"22b30d838b66fea4a4a234b4499eaf4f6c7fd939393d84ad2ed55a3e42415796","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7f146edf2855d322bb8478a24ddb4c72edcba0b749f1a73cfc64e33b042ec258","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · mmmuPro"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b","upstream_model_label":"Qwen3.8 27B (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"75c40512700fc73e46d43ff763db70700c10b0e76fe23b3bf40867de60520a35","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.300578,"score_display":"76.3","source_stated_rank":null},"run_fingerprint":"e97350e31a863aaa26d60e8c29a0b333669eca50f2fdfad0ce3ff4ec5d587c5d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e7048da490651d81baf5225efd8f52563eab616fdeed3d59b5aa5462a0c5d00b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Medium) · mmmuPro"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-medium","upstream_model_label":"Qwen3.8 27B (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8dc23cbcfaa9b0b4c78eb897e21732f010396d84a12942c5fc5eb1bc3689a04d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.16185,"score_display":"74.2","source_stated_rank":null},"run_fingerprint":"134479778e535b2a920bcd064d81fa5d188f834da61143d3b4934806fccc1be9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"743ca8a3f7089a25284b075493c11145424fcdc55eae5b66e3d167a743107974","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Low) · mmmuPro"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-low","upstream_model_label":"Qwen3.8 27B (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"549c12d30ca235d581891efca6d6be804f155e48e30da16719df6efc229e8ae9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.757225,"score_display":"73.8","source_stated_rank":null},"run_fingerprint":"74fc73e3a260c9d4d97fa8d5b9f14df2f1a80b908c3ca9e8dff726e9d496464b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"32a5f5d7afc6bdf68f17d852245f4f31995bf4565b0c8442f7703e3af58159e0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-non-reasoning","upstream_model_label":"Qwen3.8 27B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1aa578bd3d140bca894cf3ebfda410d7204c67569b5a6a75801bcfeacd481ddf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.942197,"score_display":"69.9","source_stated_rank":null},"run_fingerprint":"f013e9992d1b0ea6e159a4794ba1e796a68c54c2d0b4dda28b07584872f2ea0c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"40a8844b756faa782e0a9b5c53961b48563263f86d070b106abfd8cdcd7eda70","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 27B (xhigh)","source_effort":null,"source_model_version":"qwen3.8-27b_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"4ed6ea6e83a3e937c49a10f9d97c69165e8a46b24f7a82643d048284b6af4547","result":{"confidence_high":13.336578,"confidence_low":2.096245,"sample_count":71,"score":5.42857142857143,"score_display":"5.4","source_stated_rank":108},"run_fingerprint":"49adbe8db8ac17703a4c14fa945dc458deaa4375ebc81905e29d495a7b002d4f","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b431725b6711b9609d2c4919deeb693bb2a30e07e399d85ab4966ca0417e8ba3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 27B (medium)","source_effort":null,"source_model_version":"qwen3.8-27b_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"ffdd0a4792ea8d21a759011a478bcc1b7f3e37afc6e8862647001ca26eab3edf","result":{"confidence_high":5.132974,"confidence_low":0.0,"sample_count":71,"score":0.0,"score_display":"0.0","source_stated_rank":167},"run_fingerprint":"e44ce34afc064936149949be7dceee5dec66ce0dcac1e51e35cbb5cf555b5db9","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8ede9b1c13e083e1dda5e9fa3efd25bc2cb3f264054357b4bbf15ec8cfbe9465","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 27B (low)","source_effort":null,"source_model_version":"qwen3.8-27b_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"96cb4f43e007c02a16a3b81493f544914cf0a4b8d48636f704dc6769c4297fa7","result":{"confidence_high":5.132974,"confidence_low":0.0,"sample_count":71,"score":0.0,"score_display":"0.0","source_stated_rank":167},"run_fingerprint":"9d27bb4190bc4a80627343cb755505cd3500c5d5a242d8e9a69c321f6c283fc6","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f50d99bc2f69697129044f7f9eb012aa9b896c503aa1b5602945694d89811df0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 27B (Non-reasoning)","source_effort":null,"source_model_version":"qwen3.8-27b_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"1011febe2cd15679be44371040ee32e7febec222b245f410c7e1bf519853e4cd","result":{"confidence_high":5.661392,"confidence_low":0.013679,"sample_count":71,"score":0.28571428571428603,"score_display":"0.3","source_stated_rank":156},"run_fingerprint":"5668f12425dcbc397309201417a1eb1f96766bd6fd2ea56334dfb8136f56cb19","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"418f92f61b1a6d352b01f8d13340781534ffa1abdaea7805fa952f8afe91fa7a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · critpt"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b","upstream_model_label":"Qwen3.8 27B (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5bfda432d1a36d2864bc105101218e5180a349489f9b4f18c61c6176084e4241","result":{"confidence_high":13.411833,"confidence_low":2.082955,"sample_count":70,"score":5.428571,"score_display":"5.4","source_stated_rank":null},"run_fingerprint":"25dbd50de9aaf060abb962d3e0b308e813bd8e663c899d4f9e5270aba2b70b93","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4fc25e0ba26b2382ccd98987fb79e6d11678a9eef2da1a80a2b7175cbf5cd635","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Medium) · critpt"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-medium","upstream_model_label":"Qwen3.8 27B (Medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"fe846b19d69f98219d27664091470629ee1728d46f1aadc5562d7a3daf852625","result":{"confidence_high":5.202487,"confidence_low":0.0,"sample_count":70,"score":0.0,"score_display":"0.0","source_stated_rank":null},"run_fingerprint":"3b72c6d3f19c58a12524c631141d5f3e20f1d23f4e87c9de497c2fbd26f2e686","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"caa611f74d7d96eda308b92cc306c84fa2dfcee17c0c983e198a769fda6dfc4f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Low) · critpt"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-low","upstream_model_label":"Qwen3.8 27B (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c7e77649d5d19785b53be4b2bf6ccf45cb2bd72c991fb15b880b046e57482826","result":{"confidence_high":5.202487,"confidence_low":0.0,"sample_count":70,"score":0.0,"score_display":"0.0","source_stated_rank":null},"run_fingerprint":"a68c6c7eab4597e5666c03678e9d5656ea62ae4d8041c227c9e7351cb6ccb3b7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"37d0831093b863be17fe4ab100835766f9900871956f005abd15c022e5eba844","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · critpt"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-non-reasoning","upstream_model_label":"Qwen3.8 27B (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"e80b8a61f078f738d270234d7e6af98d785fcf72841b122d11337f5ca0605be0","result":{"confidence_high":5.730683,"confidence_low":0.013504,"sample_count":70,"score":0.285714,"score_display":"0.3","source_stated_rank":null},"run_fingerprint":"b416562b87620b17d45cab65cef428ab8778369b98c913e7afa988f84d20bdea","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9936536be52561763e88d69d3fac05c714ae13a1632e8741f688d0a3ae4ec9ab","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.077188,"date_is_proxy":true,"latency_seconds":599.776,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.553},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.8-27b"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"00b3598d4c633134a49409e74d1ef692e3bb0ccbb9cb5ce9166db623832b90cb","result":{"confidence_high":89.04388,"confidence_low":82.95612,"sample_count":500,"score":86.0,"score_display":"86.0","source_stated_rank":15},"run_fingerprint":"d339a972295d2ed054b93fd8243bad1f9ee7caf8044a65a194de8443e0c5836e","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_12e8a62d68d933c29dcdf020","split_or_window":"vendor-model-card-run","unit":"percent","version":"DeepSWE 1.1 vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"270fce12814894194fea42642329aba5171abf32533cd921bf9726121fa26c04","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Evaluation table row Qwen3.8-27B; DeepSWE 1.1 column; evaluation details immediately below table"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card vendor run on DeepSWE 1.1; temperature 1.0; top_p 0.95; 256K context; reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"54122d35ffdbd0fd3eb0527b7ca43a75f3d8c6d231a28990b2a760d842259ee4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.2,"score_display":"42.2","source_stated_rank":null},"run_fingerprint":"f4cd227ec382a01c5f7fd9b5b3525235783e443217ac6dabfaca7071d9084d66","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Evaluation table row Qwen3.8-27B; DeepSWE 1.1 column; evaluation details immediately below table","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f97c4e36649aacbbeb6c72f4","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_d4a5abde3ed9874198f1dc1f","split_or_window":"qwen-refined-vendor-run","unit":"percent","version":"SWE-bench Pro · Qwen refined vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"c223b682147e7267bfb42b0b41875dd0be267e77d3db708aa64abb8ef9ecf2fd","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Evaluation table row Qwen3.8-27B; SWE-Pro column; evaluation details immediately below table"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card vendor run on SWE-bench Pro with documented corrections/refinements to problematic tasks; task count and reasoning effort not disclosed; temperature 1.0; top_p 0.95; 256K context","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"21de62ad0e3d722b20b2e6aae8704f41a9f01f310b960337d76cb1b9a71e5fbe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.7,"score_display":"61.7","source_stated_rank":null},"run_fingerprint":"82959b804dca95af6acb0d709d5b39daf026aedb47eed4d605d2ef4782817ca6","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Evaluation table row Qwen3.8-27B; SWE-Pro column; evaluation details immediately below table","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f97c4e36649aacbbeb6c72f4","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_232b90e5b13ff602749b9048","split_or_window":"qwen-refined-vendor-comparison","unit":"percent","version":"SWE-bench Pro · Qwen refined comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-27","status":"fresh"},"measurement_id":"a7da3a6ebe30321353aa0775ed285a7bb4b6c168c20b50941dc8623813fe1e83","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Evaluation table row Qwen3.8-27B; SWE-Pro column; evaluation details immediately below table"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on the corrected/refined task set; temperature 1.0; top_p 0.95; 256K context; task count, run count and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"35ca37ccc366f1ba478dab33841a2c0c18ce53fd62f291ca05341f7728b225b2","source_published_at":"2026-08-27","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"82142319a9a411900f326e470d1e8c2967a141de1d657112d3229abe9bfbe11e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.7,"score_display":"61.7","source_stated_rank":null},"run_fingerprint":"11a4f4e19a5032aa78f3bca14a8384a86d05757ed204c758f623037709c32e5d","source":{"content_hash":"a8132c0d08dcb8936d1a9078fbf74a39072aa1d37bc0b3c112c58e2308514ae5","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next","id":"qwen3-8-flash-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen Community License model card; aggregate score facts with attribution","locator":"Evaluation table row Qwen3.8-27B; SWE-Pro column; evaluation details immediately below table","name":"Qwen3.8-Flash-Next official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_64cadcdf82035347962106ac","provider_config":{"source_id":"qwen3-8-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":4,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"secondary-re-report","verified_status":"secondary-re-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b72f3c4306918bf7245110e95f8ae11f190cf4dfae006ab4afdb9666ac5f1569","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":3.553383,"date_is_proxy":true,"latency_seconds":5924.968,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.938},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":65536,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.8-27b"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"edcbfe024cd518eeb16cd618ecda4747377961a87624c2298eb816f874d733be","result":{"confidence_high":48.73448,"confidence_low":29.377519999999997,"sample_count":null,"score":39.056,"score_display":"39.1","source_stated_rank":33},"run_fingerprint":"ba60ce2bd2b2dec3181fcd597e044b3d788caf2317cb9102f2c44d4134a56103","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d7ab9f315f3f6596b3c2ab93fb4e38c8005081c74dc0ec90ea5f00348751df22","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.044983,"date_is_proxy":true,"latency_seconds":149.298,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.035},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.8-27b"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"9ac2ce5167d2e53c88686251a0581e6f4c39b08267180aa3a06646d0be5c1287","result":{"confidence_high":86.02459999999999,"confidence_low":81.9674,"sample_count":null,"score":83.996,"score_display":"84.0","source_stated_rank":43},"run_fingerprint":"0d40e0d9c8831b2a151bafebd70e77134d1f5c43ecf5a2891a995ac2d7f52901","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_91081dfaad812c5830ccddde","split_or_window":"v6-since-2025-01-01-182","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-09-02","status":"fresh"},"measurement_id":"14fa59a5e796b33d942527f07f020b9fb2ece4ff6385b75d95444e74bcec23c9","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2/resolve/561dae0aeea1efbdafe122f2e385fdeaa99127e7/README.md","evidence_sha256":"399a0d0791ac1bfec5a2045df28d2b200a2661c0a81f2867a17dc95cd89d4986","kind":"reviewed-static-report","published_at":"2026-09-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2","version_id":"2d9354e5d0ffc5d484d1a8f95b09f83b6f4e55e4ec0903e813b0bb08f36992cc"},"source_locator":"Quality vs FP8 table; Code generation; Escha-W2 column"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Escha Labs","evidence_verified":true,"freshness_proxy":true,"protocol":"Same SGLang fork; xhigh thinking, 28672 reasoning + 4096 final token allowance, temperature 1.0, top_p 0.95, top_k 20, seed 42. All prompts in denominator. FP8 and W2 use different GPUs and engine dtype. Quantizer self-report; different GPUs and engine dtypes. W2 includes learned quantization adaptation and is text-only. LiveCodeBench is only the 182-task 2025 subset. Small gaps do not prove a quality difference.","quantization":{"backend":"Escha SGLang fork","base_checkpoint":"Qwen/Qwen3.8-27B","hardware":"RTX 4090","id":"escha-w2","label":"Escha-W2","limitations":"Quantizer self-report; different GPUs and engine dtypes. W2 includes learned quantization adaptation and is text-only. LiveCodeBench is only the 182-task 2025 subset. Small gaps do not prove a quality difference.","reference":false,"study_id":"escha-qwen38-quant","study_label":"Escha Labs · Qwen3.8-27B","weights_gb":10.15},"question_count":182,"reasoning_effort":"xhigh","result_published_at":"2026-09-02","source_content_hash_method":"raw_bytes","source_content_sha256":"3e8057f24bfaa4a5f96faca89a19de8adc8df043dc338517dda092c8f432634b","source_published_at":"2026-09-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d1f6f025478a4b21ecbb2b9e9095fcf33b7b23a85004aca68842d311df1dc6e0","result":{"confidence_high":null,"confidence_low":null,"sample_count":182,"score":86.81,"score_display":"86.81","source_stated_rank":null},"run_fingerprint":"440d476d6f2279e1e848a5260924ad6bc88349334a9e8ce79079992d1ab0a125","source":{"content_hash":"b8398566036ec78cf366b63879678897049710b9ac91d17bcad9fc24f9481c9c","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2","id":"escha-qwen38-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Quality vs FP8 table; Code generation; Escha-W2 column","name":"Escha Labs Qwen3.8-27B FP8 and Escha-W2 study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_91081dfaad812c5830ccddde","split_or_window":"v6-since-2025-01-01-182","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-09-02","status":"fresh"},"measurement_id":"ef5bd8f488e011a3bd103e9dc16b3a0b067ecd964c11e1f9d8e53481bfb5413a","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2/resolve/561dae0aeea1efbdafe122f2e385fdeaa99127e7/README.md","evidence_sha256":"399a0d0791ac1bfec5a2045df28d2b200a2661c0a81f2867a17dc95cd89d4986","kind":"reviewed-static-report","published_at":"2026-09-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2","version_id":"2d9354e5d0ffc5d484d1a8f95b09f83b6f4e55e4ec0903e813b0bb08f36992cc"},"source_locator":"Quality vs FP8 table; Code generation; FP8 column"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Escha Labs","evidence_verified":true,"freshness_proxy":true,"protocol":"Same SGLang fork; xhigh thinking, 28672 reasoning + 4096 final token allowance, temperature 1.0, top_p 0.95, top_k 20, seed 42. All prompts in denominator. FP8 and W2 use different GPUs and engine dtype. Quantizer self-report; different GPUs and engine dtypes. W2 includes learned quantization adaptation and is text-only. LiveCodeBench is only the 182-task 2025 subset. Small gaps do not prove a quality difference.","quantization":{"backend":"Escha SGLang fork","base_checkpoint":"Qwen/Qwen3.8-27B","hardware":"FP8: different GPU, not specified; W2: RTX 4090","id":"fp8","label":"FP8","limitations":"Quantizer self-report; different GPUs and engine dtypes. W2 includes learned quantization adaptation and is text-only. LiveCodeBench is only the 182-task 2025 subset. Small gaps do not prove a quality difference.","reference":true,"study_id":"escha-qwen38-quant","study_label":"Escha Labs · Qwen3.8-27B"},"question_count":182,"reasoning_effort":"xhigh","result_published_at":"2026-09-02","source_content_hash_method":"raw_bytes","source_content_sha256":"3e8057f24bfaa4a5f96faca89a19de8adc8df043dc338517dda092c8f432634b","source_published_at":"2026-09-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"092982aac0e3fd6a15d8f67d6c8203570aea8863533a1275db7a5755d6c21cc7","result":{"confidence_high":null,"confidence_low":null,"sample_count":182,"score":85.16,"score_display":"85.16","source_stated_rank":null},"run_fingerprint":"1bc86aaf742b224cf4fd7411a2fbce2167e7e90c90675414f5fb3c980969849a","source":{"content_hash":"b8398566036ec78cf366b63879678897049710b9ac91d17bcad9fc24f9481c9c","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2","id":"escha-qwen38-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Quality vs FP8 table; Code generation; FP8 column","name":"Escha Labs Qwen3.8-27B FP8 and Escha-W2 study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/EschaLabs/Qwen3.8-27B-Escha-W2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_b530ca4da413885c54f52baa","split_or_window":"qwen-vendor-reported","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:49:13Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"3402bdf95bfcd1ad0497a5c72a7a2dcba3ac0bdd5667f8f1b1366f502959737f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Benchmark table; livecodebench; Qwen3.8-27B column"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"LiveCodeBench v6; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ecf2e38af773776ffb2bb1f6a191e6895f5900824635c82a4f799243e47caf4a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.3,"score_display":"90.3","source_stated_rank":null},"run_fingerprint":"0e15668f5e77a13018eb44ccc985f491a56d7b5f1c3e0613c70b5005b004b2d3","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Benchmark table; livecodebench; Qwen3.8-27B column","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f97c4e36649aacbbeb6c72f4","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bef34df42c6a953eed58f2ba85c46f2cefae1cddb583b6eea9c355b6ee1e738f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 27B (xhigh)","source_effort":null,"source_model_version":"qwen3.8-27b_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2747539c51e8676bcb7c366c24be03b7e8ed7bd2e01030236437d86fb34e2256","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.6435185185185,"score_display":"46.6","source_stated_rank":117},"run_fingerprint":"7a5e376df9bfb3654eb82b3a58a01dd0c10bfe9cec2657987e2abe876161d054","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fcc9fe9ef7f111cbff4d9bbc96adc57150da720290973b9b315102ee92ddc361","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 27B (medium)","source_effort":null,"source_model_version":"qwen3.8-27b_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"799bff0b5025f7e75a3e45e93bbf6f0a41c3dd16fc40d55f2cad0ccc889a2912","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":38.078703703703695,"score_display":"38.1","source_stated_rank":160},"run_fingerprint":"9eaa4c10a482dc2c54fcaf5c14103606d7683d2a30ceac97cadc63e6f9a58550","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5e6222e8a26ba6f6e6255b015cd957919b4b8cb52a3a6e68c780e59c224c5566","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 27B (low)","source_effort":null,"source_model_version":"qwen3.8-27b_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fc37814c298adf0e84a3405a5a71559434c8eeffe89724234245f9dcc4b7d6e5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.814814814814795,"score_display":"39.8","source_stated_rank":152},"run_fingerprint":"fb3fb5909e635c55e1fac6330ad09bfa60d65fafa7cc98a807d80baa926522d3","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c2923f495b164cc808b650bb93c434e71a912423c4b0ab2b321da4ce07ee118e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.8 27B (Non-reasoning)","source_effort":null,"source_model_version":"qwen3.8-27b_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"afb13e20ea30994b90030d3c281ef3e2062ab342da7e7964e41808298db2264d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.6481481481481,"score_display":"35.6","source_stated_rank":172},"run_fingerprint":"1a6f28e9c81cfcd55ba698091c66c953773434618cef4e9b9e30fa401fd797a2","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"578abcb1f719cd028a5b16151813c86235f179caba36e2d12e0f616adb927d39","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · scicode"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b","upstream_model_label":"Qwen3.8 27B (Xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"58e255d4bad92915dfe33bff6b3a9b02f6a169e4d302442e58a28f5f7e388695","result":{"confidence_high":52.411504,"confidence_low":40.963899,"sample_count":288,"score":46.643519,"score_display":"46.6","source_stated_rank":null},"run_fingerprint":"594372d73ed8ebe42ccc509bacdcb79fffefc7eb49c2fd103a85d0a57996d38c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5c7235b7eb6e74854e15ce70c71a715ad6d631a3dc12351a0062ebb6392a6578","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Medium) · scicode"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-medium","upstream_model_label":"Qwen3.8 27B (Medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"826a70a4a3c12b8f0f316183f8a8931f30f1f494e7a2e810b4a4d6bd6fb0a32e","result":{"confidence_high":44.747382,"confidence_low":33.551349,"sample_count":288,"score":39.00463,"score_display":"39.0","source_stated_rank":null},"run_fingerprint":"64c10d3b3c533dcfbf6a02dace4333693adad1c1038a6517b84b7a04c57b0e94","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0228d2c2bf38ed16d0859cbaa3e31bfd51ce211c1627b478777c9a8ff1dcbd3a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Low) · scicode"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-low","upstream_model_label":"Qwen3.8 27B (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"9526fc853207a056580ab005dc8ae02d9003af7728390e8f89644ecc227bcf7d","result":{"confidence_high":45.800598,"confidence_low":34.554041,"sample_count":288,"score":40.046296,"score_display":"40.0","source_stated_rank":null},"run_fingerprint":"018fddcc4f299c439b22d7d669f7298d5031f6789d44d8eaaa4f3866263c30c6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"025443b492d90447890ca1ab7c522015bb3fee28c106d4bdc8f699937cd70f56","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · scicode"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-non-reasoning","upstream_model_label":"Qwen3.8 27B (Non-reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ab7817d8c3fa2902a2e3b130ac41e85aee7e7546a923a35f99e802c5e3b3dd3f","result":{"confidence_high":41.925766,"confidence_low":30.890539,"sample_count":288,"score":36.226852,"score_display":"36.2","source_stated_rank":null},"run_fingerprint":"1eedc8d0ad3a12c950bc68c4d35d28e072467c8739ae875a7724ef99b6ccb3fa","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"824087c7e954253133e00751036bf28f0b75fe07569e26726cc942a00795069b","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":23.456372,"date_is_proxy":true,"latency_seconds":2619.407,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.825},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.8-27b"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"9afdd0d1a1fab511cef7428358903d98c368a5e30b3416c47b2ee9f68242a93d","result":{"confidence_high":74.30699999999999,"confidence_low":55.392999999999994,"sample_count":null,"score":64.85,"score_display":"64.8","source_stated_rank":40},"run_fingerprint":"733a98e5a94af69a921499249d1240fef39f30c50fbbe774afaf27961f96c186","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2eb702f0cf518144c8f09cf38a48d64debb69ed0b9d154214101b6b432e469f8","metric_details":{"license":"Apache 2.0","variance":12.765577015656515},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.8-27b","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"eb2dc50fb2c31599a31c2986e6cf5e4a5ae6becf9341a1c00728e15e88354c53","result":{"confidence_high":1598.2378390217102,"confidence_low":1584.232348894186,"sample_count":12388,"score":1591.235093957948,"score_display":"1591","source_stated_rank":28},"run_fingerprint":"2458ed0169c3a50f77ea7c7627577c6333ad9a161cb781b568f102d0308314d7","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_457b335e7d9b4a78950ca289","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0ba0ccccdd10bc3d5085c3e5497b0cb88797cc886969021c7e9880bb40ef8038","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-08-14","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.8-27b","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"ed205133f979f93a737c189b370ff57e51bccfb312c493c6af598ae5a44d0bd9","result":{"confidence_high":1602.77,"confidence_low":1583.81,"sample_count":4913,"score":1593.29,"score_display":"1593.29","source_stated_rank":21},"run_fingerprint":"db046b5fde03c16b7001b5484a70ca1e33865bf85d65700d9847ef7361e7966e","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3674a647f641ca878bab80d9","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0f702a1e007f8d1c87c10549dba50411ff3ea2fedf73da2ef0fe4354ba6f7f37","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.677989,"date_is_proxy":true,"latency_seconds":892.585,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.254},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.8-27b"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"61670fba61fef6e6d703e8f54ee9e9b69b32c78080dc888bd932fd640961d25b","result":{"confidence_high":68.72484,"confidence_low":48.12916,"sample_count":null,"score":58.427,"score_display":"58.4","source_stated_rank":41},"run_fingerprint":"207ad7439d4fecde14460ead4e627529a262333ebb21f7ebd7d97a514555b798","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9a2ef07da9b9627d5bb1e21ace93a7bf6dcf70da6812de58e668359cac1cab2c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 27B (xhigh) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-27b","upstream_model_label":"Qwen3.8 27B (xhigh)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"b52288d36fa0e78d41c118a290f91791820c7cbeb0b8283613915ad6ec0c5eae","result":{"confidence_high":86.806323,"confidence_low":70.280156,"sample_count":89,"score":79.775281,"score_display":"79.8","source_stated_rank":null},"run_fingerprint":"61b113711d2867c0fd211f7b43f7edc7f2c286b57ab23852d8bab56d32cebea7","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (xhigh) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e7dbdaf4b3cd76db19f1d0d7e00fb7c66edd9c108dc493b5147929db54b13177","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 27B (medium) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-27b-medium","upstream_model_label":"Qwen3.8 27B (medium)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"3d7dfa5f6f335b008e57610441e035f08e344d6e8b63d460d826f2368e3fa4f6","result":{"confidence_high":74.252667,"confidence_low":54.829124,"sample_count":89,"score":65.168539,"score_display":"65.2","source_stated_rank":null},"run_fingerprint":"46140e6a5d95f2b224a7a07a1c6ee9c3c2b09ed99e86ab63c9154056bf28ab25","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (medium) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ef13e3de492c9ce96ea0d884e34973397355fe7cb02acc5579a4bcbd69c0c36d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 27B (low) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-27b-low","upstream_model_label":"Qwen3.8 27B (low)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"1b1aebba74c41a1d0c11545e783e692c0b9d1d7f2ff4b14a5f3b03f85e05b52c","result":{"confidence_high":76.256171,"confidence_low":57.134033,"sample_count":89,"score":67.41573,"score_display":"67.4","source_stated_rank":null},"run_fingerprint":"a886f07c217d1945d7028bb8e86126e0701a8d281fb3ef16ff20dd2718b9c77c","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (low) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"45e92b26bf1aeeae64cd3eb21e4e4dcec0344ff55f730e9cb7d60f3634dbd9be","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-14","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-27b-non-reasoning","upstream_model_label":"Qwen3.8 27B (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"36e1c1c4f3e0025d1fc7ef6905135974a2b75ae769502dc84c597bf2efd0858e","result":{"confidence_high":59.271496,"confidence_low":38.933331,"sample_count":89,"score":49.06367,"score_display":"49.1","source_stated_rank":null},"run_fingerprint":"e99cca25bc93b09980aaf22a92d5d6e01b58c884931d59a9ce5cf4cb4db65b87","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_6b8b8501788df9ac3602732e","split_or_window":"terminus-vendor-run","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"7cfaa9054e13d19f125b7611630a2348d40f74a472793c57c88b6a853b48cdf6","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Text Performance table; Coding; Terminal Bench 2.1 (Terminus); Qwen3.8-27B column"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card Terminal-Bench 2.1 run with Terminus; exact task revision, run count and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Terminus","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"3dcb100d6dc60e6a13a4f3f14d5b584f348b0299bb0c8eb0a2dc896328be26c1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.0,"score_display":"73.0","source_stated_rank":null},"run_fingerprint":"cc787f985086a96ac288f600a2fd3393f2ba78562770acc4b20ea9dded8ec9cd","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Text Performance table; Coding; Terminal Bench 2.1 (Terminus); Qwen3.8-27B column","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f97c4e36649aacbbeb6c72f4","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bfed9518e7d61041954d52cc49b3f37e6fa25fc1ea030b5ba7dbce472a56b725","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · tauBanking"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b","upstream_model_label":"Qwen3.8 27B (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"86c01ac4285f5343ec16e722bb80c398979d1aeb68c88a4ddf97731a40a78087","result":{"confidence_high":57.867671,"confidence_low":38.364043,"sample_count":97,"score":48.041237,"score_display":"48.0","source_stated_rank":null},"run_fingerprint":"d6df3164b10bd99fe4ff0c8660e4e7593f3bb944391d860dc33311edcef05c94","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0be123e330dfbc3220740ecbcc201421412ac512031c1b0b063a29f90f43f86a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Medium) · tauBanking"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-medium","upstream_model_label":"Qwen3.8 27B (Medium)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"3a1e4c7dff16b05c887e6a26f2e870b08e14664b51c7c40982b36df93c3b1634","result":{"confidence_high":57.267405,"confidence_low":37.774323,"sample_count":97,"score":47.42268,"score_display":"47.4","source_stated_rank":null},"run_fingerprint":"9eea1c973ff125e64a06faef3a841faffd258820154027e856290dce20db699a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Medium) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ef836a34d331939518b1997e0655f6b97467b58afdc15c33634deedd6db8108f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Low) · tauBanking"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-low","upstream_model_label":"Qwen3.8 27B (Low)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"ab4181aabe771a452419f4b6bc9827fb39b7268d986f0d5440aaae03f6e890b6","result":{"confidence_high":41.986721,"confidence_low":23.702042,"sample_count":97,"score":32.164948,"score_display":"32.2","source_stated_rank":null},"run_fingerprint":"2f71f3de2196b1ff759ebff23552369c284bfb28b4fa52ebe8fff33f8ee1cf93","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Low) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"39bd13457ba4d2dc57fdbbca0e7a28dd4fe77c96d1896cf6307e6581a1be2b2d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-14","source_locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · tauBanking"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-27b-non-reasoning","upstream_model_label":"Qwen3.8 27B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"f4c3bf9d316140f7eb14993b03bdb5669ec8e8c18878aa90132fb8756314f9b1","result":{"confidence_high":29.033283,"confidence_low":13.25244,"sample_count":97,"score":20.0,"score_display":"20.0","source_stated_rank":null},"run_fingerprint":"450ab298e067c99a64d759682f6340437d91a5bc912f6fe8e042c71f7236a9df","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"90515179911bc2a6a158039971a52341cdb603a5f50299bf5f67fae22c25d28d","metric_details":{"confidence_level":0.95,"license":"Apache 2.0","session_count":57343},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Qwen 3.8 27B","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5d36e7c685853a13bb81a3d216f2e58c7ebd876b3dd0970215e19a924a0635be","result":{"confidence_high":-0.010603095775493641,"confidence_low":-0.022521536082842943,"sample_count":6972838,"score":-0.016562315929168292,"score_display":"-0.0166","source_stated_rank":37},"run_fingerprint":"a8d16a606c64fdbdb6c693a64f76b941761123c15c86e01436096ceebc95263f","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8d97cd82dc667edb7aa71a27","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"007460b89c4f1ef67f8c225672914a518f6339f41b61a2e5c6f6deb6157197b5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · gdpval"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-8-27b","upstream_model_label":"Qwen3.8 27B (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"03d60611089178154c414e3404b1b295dcd89408182920e1438a3b94317cf9ae","result":{"confidence_high":1444.94,"confidence_low":1401.48,"sample_count":220,"score":1423.21,"score_display":"1423","source_stated_rank":null},"run_fingerprint":"7d25fcc54ae3ac714be444651f9198d81dbaf3011a3c80c663b377e626d987ab","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_6037c094ef27884dccac420c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4a0ec1cd34fc35ade5aad35ecafbf9fa3b10823164655aea2961babeb7adb496","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Qwen3.8 27B (Medium) · gdpval"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-8-27b-medium","upstream_model_label":"Qwen3.8 27B (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"e48dbc8b57ab2190f52fdc15a4f784c4e98116b6f7f9deae5accf49819c4859b","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1368.49,"score_display":"1368","source_stated_rank":null},"run_fingerprint":"1bd9dd9433210e17cac09afa803c299a595091fdc3996a635844dc8a0a0b7c54","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_94fa1fa553b6d2a6412e26e3","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8c3e1e5a96de971e42d16f005be0657aa4a40c3f6b95be2277303ed33a407a50","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-14","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Qwen3.8 27B (Low) · gdpval"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-8-27b-low","upstream_model_label":"Qwen3.8 27B (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"61ac5fa717b2e910eae46c37426e7068d6705e0f903381df0375791c62955c44","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1341.57,"score_display":"1342","source_stated_rank":null},"run_fingerprint":"f99d104dc0aaf49fef50ac3b1d24d2820eeaaaee2cd2cc1d16c4ba482c8fdc0b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_4cfdaf1c2c69a6d20ec25fdf","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8a273a63db027231311c0c270812703c65c4c0e33d51a0ece4903c3c0d5cf081","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-08-14","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-8-27b","name":"Qwen 3.8 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-8-27b-non-reasoning","upstream_model_label":"Qwen3.8 27B (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"b88501e5ec039ab74232b3be54b18bfa545c175c615321c7749e9d99df6f0532","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1052.21,"score_display":"1052","source_stated_rank":null},"run_fingerprint":"a9edad6f46d585cf79fda2546c16e373b3f2164d32e04e1a266f296e96fa9d5d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8 27B (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aba0c520efad9ce6d4c8a1de","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bb0a422b134f59fdb54513c8f7f2912b03c3bdd992d5f1ea79fa8dc33ab5b1e8","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-20"},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen 3.6 Max (Preview); model release: 2026-04-20","source_accessibility":"Unknown","source_model_name":"Qwen 3.6 Max (Preview)","source_model_release_date":"2026-04-20","source_model_versions":["qwen3.6-max-preview","qwen3.6-max-preview_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0729e15f91672b5ddd22180fac3d96127e91987b70fc52c3726a319ae6cca3a7","result":{"confidence_high":151.99,"confidence_low":147.57,"sample_count":null,"score":149.24,"score_display":"149.24","source_stated_rank":55},"run_fingerprint":"457f32d349a333d26b58e8d205f1311580172f84747fab88fbeb7f69db188918","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d3b3c1e009228ac3e1f855f2","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"587893c3210225fbbbf93e411473ad7de7c4988f558fddf78937e594ac97e36e","metric_details":{"license":"Proprietary","variance":17.526900464230724},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1468.5005252432952,"confidence_low":1452.0896914573175,"sample_count":5406,"score":1460.2951083503065,"score_display":"1460","source_stated_rank":59},"run_fingerprint":"35d4babd259952f520dc2f0b4126e664bd677fb9910495b9e0b642254c77a724","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0cdc415c5b29e9a4e97c44b","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:33:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:33:58.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"76a9bb2f972d331fcce3e692317f8a4ba87b4f570ae6b15a51d57682df1e73b7","metric_details":{"best_score_across_scorers":"0.5200400801603207","model_release_date":"2026-04-20","stderr":1.5822434816352415},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"iyPiR8GUuyNNA6Fssziq32","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FiyPiR8GUuyNNA6Fssziq32.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/iyPiR8GUuyNNA6Fssziq32.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d333fa9c9c40094adf12bb365e2ca5af71c35f022bcf022cf3ed1e8461311ff1","result":{"confidence_high":55.10520524003714,"confidence_low":48.902810792027,"sample_count":1000,"score":52.00400801603207,"score_display":"52.0","source_stated_rank":22},"run_fingerprint":"b823f120e9a5b69c7d29c3521881217b435a3fd11706de126b8b1ce726c970b8","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3fc6875ee3cff2a0aacf3f9d","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:46:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:46:39.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3a03463fbb7c9c4bab3644ea982287b2af1bf813f9a0ed9d7a4a7ad187aa6b73","metric_details":{"best_score_across_scorers":"0.8737373737373737","model_release_date":"2026-04-20","stderr":2.3664359402880177},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Jv2hJvjpvA8xJW3Bzv7bfw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJv2hJvjpvA8xJW3Bzv7bfw.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Jv2hJvjpvA8xJW3Bzv7bfw.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"0b0b7941b04d488dd65911e62a5ef48434dbfce3fef6c7a28590d294277391d3","result":{"confidence_high":92.01195181670188,"confidence_low":82.73552293077286,"sample_count":null,"score":87.37373737373737,"score_display":"87.4","source_stated_rank":71},"run_fingerprint":"443d2d1abfd159780b4a95ddcaa0b3c833b34c8947cfcbc73cf43f8f6833a21b","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c656be424b91b5bbd75f2aec","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:42:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:42:24.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c6a0fdb795b7a6c4b31a21f6bcecd11b09a9333248699ce8ddc3d4537a905b8d","metric_details":{"best_score_across_scorers":"0.9111111111111111","model_release_date":"2026-04-20","stderr":4.290254662948546},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"EfJpx5HztMmL5EQSwsyqMe","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEfJpx5HztMmL5EQSwsyqMe.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/EfJpx5HztMmL5EQSwsyqMe.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"53721f0f1d9a2b4915003b28b41904796c4720cefb18cc3ec8c3c5237db1499a","result":{"confidence_high":99.52001025049026,"confidence_low":82.70221197173197,"sample_count":45,"score":91.11111111111111,"score_display":"91.1","source_stated_rank":65},"run_fingerprint":"7794308224aa72817885d0e3def41706bba05352bffb9f544f8fac608e8c0787","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4b807b8f467caef3423aa98d","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ae3ca62e6cdc5648731a2ea366e4b2fa53d1415ac52a0d6df91503e0456ffabf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.6-max-preview","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"6dbccd280fb64a450c00c0ff2232ac25f842fdf586a8d550543ba0310005b1a2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.0,"score_display":"63.0","source_stated_rank":20},"run_fingerprint":"ab9534a9dc1a82c4e695d1b6a92dd448dcfc7d64df62946db17bb7ab2038a112","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f0189e6e512a7f8f6c14d55a","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:50:10.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:50:10.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"564526a8f3c5df0f29868932398cf5716a083541489eda161730b322bfd179fa","metric_details":{"best_score_across_scorers":"0.2","model_release_date":"2026-04-20","stderr":4.020151261036848},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"i5cWL9DPrmhvacKyYf8gxE","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fi5cWL9DPrmhvacKyYf8gxE.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/i5cWL9DPrmhvacKyYf8gxE.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5b7be89347571cddf0205db4e2d00c3dac71d54f579a4d5463453ea6ff8494e3","result":{"confidence_high":27.879496471632223,"confidence_low":12.120503528367777,"sample_count":100,"score":20.0,"score_display":"20.0","source_stated_rank":87},"run_fingerprint":"c46b305be80cc1bf9b60141ff3329f371b0332efe10d1926ecb2a35a7c48b3e5","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bc10e1a5c32fdc3da1dfcaf","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fe0d4713a95e6e466c8f07b3885121e68aa10cb9e90c2749266f135e5093dd5d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.565093,"date_is_proxy":true,"latency_seconds":1079.228,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.992},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.6-max-preview"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"5055ef15ddaeef6aad315f47da1f3456e145100a37d17388e4f8a49bbab61e72","result":{"confidence_high":76.70432,"confidence_low":68.89568,"sample_count":500,"score":72.8,"score_display":"72.8","source_stated_rank":54},"run_fingerprint":"e2bb3563178826f5109ed4684ad631c42b7423053cea46b3d4ef1d34b61daca2","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1d20a31e24960743cd851fca","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":130,"at":"2026-05-28T10:24:53.000Z","basis":"evaluation_started_at","evaluated_at":"2026-05-28T10:24:53.000Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e2e83b677a733a25a14b470c31438a4dc4b1c0a0a0d7a388c6ff011892db4e7d","metric_details":{"best_score_across_scorers":"0.7665289256198347","model_release_date":"2026-04-20","stderr":1.924895909142592},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FkDMVLdEnJDR5PyUCT6BNQ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFkDMVLdEnJDR5PyUCT6BNQ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/FkDMVLdEnJDR5PyUCT6BNQ.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"53eba270c377fcdcd8c2f1748560836e0db70f666584c31b21c31e93add639ac","result":{"confidence_high":80.42568854390295,"confidence_low":72.88009658006398,"sample_count":484,"score":76.65289256198346,"score_display":"76.7","source_stated_rank":9},"run_fingerprint":"c49b80ade1cd87c0cb6b17b17ae86b4f4161010649d82d3d00bf3a3f2a62d6ae","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_da81bc30e02215025d6d12fd","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e76c0c329be9102ab60491b8edb41389045a4e758757255e5c497dfafc0acfd4","metric_details":{"license":"Proprietary","variance":40.69278039489152},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.6-max-preview","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"57963b1d66631eac9a091b1095bb2fe8e6cd107825f4941c4e6350504980f898","result":{"confidence_high":1494.0412100607668,"confidence_low":1469.0356394664752,"sample_count":2716,"score":1481.5384247636212,"score_display":"1482","source_stated_rank":59},"run_fingerprint":"05459d6cecf1b5cde0f87c319e03dd8ce18d2454033d10f0bb152a03c2b8542a","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a0689db8baf849f97eac474b","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4bbe8dff8a2fd24a3868631aff55a263026e481726a4611cd5b99b3497e27dcb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.6-max-preview","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"276738d2c0dc59360640c533c4d7d2f31db8bcaa284ab30efc227409ddb6cca4","result":{"confidence_high":1491.82,"confidence_low":1466.3,"sample_count":2601,"score":1479.06,"score_display":"1479.06","source_stated_rank":58},"run_fingerprint":"2e0aa21d70cd76c68a82518bf7714b109580caaa0c06ee8bb4ef54210efb18a4","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_39399e3b1fbfdd9c3a394070","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"61e4868f53d05b3bb2c80cc710028edbe85ec4fda956d1d574ce26f3d751f748","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-20","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-6-max-preview","name":"Qwen 3.6 Max Preview","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen 3.6 Max","source_effort":null,"source_model_version":"qwen3.6-max-preview","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"ed54e35bb086ed6ae1376be374c47344eb3021e2ebf1fbbe3d1e4469cf6bafed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4254.1866666666665,"score_display":"4254.19","source_stated_rank":36},"run_fingerprint":"997f1d86b2784d670a4e318d6ce7ec80b5aaf272d81b8fb087bffaba69a68cfe","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_a08522d91e6bb53b70645ffa","provider_config":{"source_label":"max"},"provider_mode_key":"Max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8a19c168e4c5857cb5a03adebad8ab9b8e069e6bf76d12db5956abe8f106d88e","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-17"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Grok 4.3 Beta; model release: 2026-04-17","source_accessibility":"Hosted access (no API)","source_model_name":"Grok 4.3 Beta","source_model_release_date":"2026-04-17","source_model_versions":["grok-4.3_high","grok-4.3_unknown"],"source_reasoning_efforts":["high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7c1eabe93edb8e6d7fb313d6a6fdb94c75eac45be1f833c3828121b98a33c95e","result":{"confidence_high":150.75,"confidence_low":147.55,"sample_count":null,"score":149.15,"score_display":"149.15","source_stated_rank":56},"run_fingerprint":"6db062ed7cc0adb392bddbe7ad958b1fbd38a2f2f5b695eea992d16bdd1902da","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_59e6bbd0b1c44e7d3a4db8e3","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5e4682d7a9477868942f5db4035ebc175e4ae34ccaa6231ba54594bbc9bf11dd","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (medium) · intelligenceIndex"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-medium","upstream_model_label":"Grok 4.3 (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"01a6ba329fea6943984c0cb38b39337485ec4a2643136831bc469f22132de503","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.781694,"score_display":"24.8","source_stated_rank":null},"run_fingerprint":"fdf5c00dc48516ead680a7aa59d95e53a52a3080106705fae0a39b44909d6aa0","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_1c8c583aa239a9f12d2052a4","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"13c97b231ff30abe0a1d4b8caa3ebc588f764735f8c0c91188c452c30ff2587b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (low) · intelligenceIndex"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-low","upstream_model_label":"Grok 4.3 (low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"f46c8f5419a97dae117bca004ed1cedf9d3c8e7e2aa8073f332d335d03c86aa0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.2959,"score_display":"24.3","source_stated_rank":null},"run_fingerprint":"a47bf58175e39e2f23a2996269e0d7c15c830244d7f879338cd31e53d4e054ae","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_7f815d75f75f06091a49b306","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"87cd939e4de4f8ec8bed13a09ed46713457b8984021e21427477458032afc350","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · intelligenceIndex"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-non-reasoning","upstream_model_label":"Grok 4.3 (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"358d51a937256f4094a47d2e91f8c98fe3addba3da7ca13f00f0a343dcf1d442","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":14.540706,"score_display":"14.5","source_stated_rank":null},"run_fingerprint":"6ae14c53db27797f4966e353d06ab5089d2d5f55267385f85d1d284c3ea347c2","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"19b73bf394f456bdf13ae69843cfc356b6e08b76b5100fc62f3efe4af5730c43","metric_details":{"license":"Proprietary","variance":3.4440262613331876},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1445.0508308450212,"confidence_low":1437.7761940677517,"sample_count":71150,"score":1441.4135124563863,"score_display":"1441","source_stated_rank":94},"run_fingerprint":"5a290ee02fbfecc9e3850ef59437f49199f3b0b21f33249600c3eed225342a4a","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3c0b61d8f0225024eaa77540","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"bc71ecf68b5683f684065bb9c7d419efecd1db3ffee7fba2c2da14720a378609","metric_details":{"category_scores":{"Agentic Coding":18.535333333333334,"Coding":69.9325,"Data Analysis":55.77266666666666,"IF":62.75,"Language":73.58,"Mathematics":84.339,"Reasoning":70.822}},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.24735714285714,"score_display":"62.25","source_stated_rank":66},"run_fingerprint":"d029ccea3e6e9ce18665f4945e4a2cfc047c2f5e78e6f16de24aa7729d345ad7","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_696ee2e03f20836e5b0f3f47","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":171,"at":"2026-04-17","basis":"evaluation_at","evaluated_at":"2026-04-17","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c630b7051d48f2911cfae02c145a3332b085e2446bd5df0032ebc6bb79681856","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/xai-grok-4-3","source_model":"Grok 4.3"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d0d156c8a19ab9488a31e274e6e36ac05b2a97e639a4aeb7a6342e19708aaf5d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.12,"score_display":"33.12","source_stated_rank":22},"run_fingerprint":"360fc8c58b83e4b4a9c6f42d710d9098d81653226153dfce86fae730a4205acf","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d2089c665b18a125cc91267e","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d320262d6c4c44fa2fdd532b1aa58c4d773d85ac6e7acb17963e5ee10f265a76","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (medium) · hle"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-medium","upstream_model_label":"Grok 4.3 (medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"eefdd6e9cbbf4533f92c58fad483574dcaf1f14ad34508ad557dac64f444dbdb","result":{"confidence_high":31.948786,"confidence_low":28.085288,"sample_count":2158,"score":29.981464,"score_display":"30.0","source_stated_rank":null},"run_fingerprint":"f61bb65e81d1b4406a2ac544a8d0ba50ac3639d4ef02152c3ca82b45d59ae7de","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_1c8c583aa239a9f12d2052a4","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cc5c4a225f588a6ff9a888f463c66fb9a7aa3149dbe6dc9c2370f74ddd277005","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (low) · hle"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-low","upstream_model_label":"Grok 4.3 (low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"91e3b2d322922d42c3adeef82c0e8c2891fe70505f9259bc3252227162342ba7","result":{"confidence_high":20.039246,"confidence_low":16.773886,"sample_count":2158,"score":18.350324,"score_display":"18.4","source_stated_rank":null},"run_fingerprint":"36d0ba9af3c37b50c7e971b32f5959a0c28e523b74ef23a8312d9d47ee6b8206","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_7f815d75f75f06091a49b306","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c5174a1341397ee9ea216828f31bac1923ad2f378767ef3bd4f9bfbde76e323d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · hle"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-non-reasoning","upstream_model_label":"Grok 4.3 (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fec9acc5ee26b972e327f85448da26c4747428bb6ffb504bc0307d4d301ca81f","result":{"confidence_high":7.903861,"confidence_low":5.780843,"sample_count":2158,"score":6.765524,"score_display":"6.8","source_stated_rank":null},"run_fingerprint":"0eb9af1f7d5b4c9299c3311397ec723c4f7dc67fb96ad91070dd55016d7cfd26","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:29:33.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:29:33.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"22c7386366e8549453ee8f5da8a754138996239ca1a778045e0b56aa5fe418b3","metric_details":{"best_score_across_scorers":"0.332","model_release_date":"2026-04-17","stderr":1.4899597242811566},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Y39ZRp553ULpUDBV2XrBbc","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FY39ZRp553ULpUDBV2XrBbc.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Y39ZRp553ULpUDBV2XrBbc.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"7d138963618e04aa90455cf99086e6be055208c460d08e4d47a196a53d8b3ae3","result":{"confidence_high":36.12032105959107,"confidence_low":30.279678940408935,"sample_count":1000,"score":33.2,"score_display":"33.2","source_stated_rank":58},"run_fingerprint":"eac3b6cdc9f237b3fd802364b1ad99a63e3e476abd8c13aefbe5074c001ad612","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"198eee94a675a153551b65b489c7c6db47811ba84d8d61f53f8295f2b63de692","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.025366,"date_is_proxy":true,"latency_seconds":739.636,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.406},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":92.929,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":null,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.3","zero_shot_accuracy":89.899},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"c1cc2cff70aae91666e400fd05a859acbbca13ba8df0889712a20803f53e0fcf","result":{"confidence_high":94.16976,"confidence_low":88.65824,"sample_count":396,"score":91.414,"score_display":"91.41","source_stated_rank":22},"run_fingerprint":"6c0bb0f749268d2e46d105035543780c9ac2f4496b5b31ee2b8d302a8fbd9f07","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e6219133f7d970918eb71d3a","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"369b83641f7c230020fd88eebb958016220603e76d657c5fd894be6c3d5f1161","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (medium) · gpqa"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-medium","upstream_model_label":"Grok 4.3 (medium)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"31955cdb87f0b9cf55f0184216c92863a22674607f047b2a0cc8db72c43beb61","result":{"confidence_high":92.629451,"confidence_low":83.866177,"sample_count":198,"score":88.989899,"score_display":"89.0","source_stated_rank":null},"run_fingerprint":"0e07fbb7882309723ffc5973ef4a4e7e8374bc5e8d810886edd39e92fda52802","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (medium) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_1c8c583aa239a9f12d2052a4","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"749b8c204642e9115b4ed9d093abc2e26f6707ecbf19022ec32cc6ef1c08b19a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (low) · gpqa"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-low","upstream_model_label":"Grok 4.3 (low)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"249765a4d4d22760091fd52e365672fc978fd422551dc707d251461784099f24","result":{"confidence_high":88.745528,"confidence_low":78.634041,"sample_count":198,"score":84.343434,"score_display":"84.3","source_stated_rank":null},"run_fingerprint":"703ea1aac0b5004845867a4e55128836e827445d4c6fb707b3a82712d88b5825","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (low) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_7f815d75f75f06091a49b306","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d2a0c3e25544f1c85f98d7f7ef21b4ba31af5dd0d11be8c635af0b556b40a23d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · gpqa"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-non-reasoning","upstream_model_label":"Grok 4.3 (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1f76786c26ca3b41180c28d5b17d3b36a1e1244bd8c2b2e0443c712ebd802db2","result":{"confidence_high":72.010983,"confidence_low":58.904349,"sample_count":198,"score":65.757576,"score_display":"65.8","source_stated_rank":null},"run_fingerprint":"7ac7e6b18f9b73025b16ccf301b59578136f2ed53c1be06f485ba38b05dfbead","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":109,"at":"2026-06-17T16:44:33.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-17T16:44:33.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0139fde40de391508337b5457cad8e652caaa6c844a7c96765ba49ec19401236","metric_details":{"best_score_across_scorers":"0.8882575757575758","model_release_date":"2026-04-17","stderr":1.9638404200482171},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Vw7spGUnJyf3vuzSjTcj2F","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"dcad9f2fd2f816561b16f7950a8fa645b0665eb616bfc4eec1c629c2e770045f","result":{"confidence_high":92.67488479905208,"confidence_low":84.97663035246308,"sample_count":null,"score":88.82575757575758,"score_display":"88.8","source_stated_rank":55},"run_fingerprint":"d6046eb9050a61542b4c7d0d39286709977473972119fb6e8cda8a7fa9b612fa","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"eec19cd234a513643c5260a8345103097e9b6b68547a14e48c50457f972cff96","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.009489,"date_is_proxy":true,"latency_seconds":637.088,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.33},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":null,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e8f7d9ed45b4d00a53c7e6749d620eee0881ac5b28b1490e9a4f84b75e4c2a82","result":{"confidence_high":86.48479999999999,"confidence_low":85.1912,"sample_count":null,"score":85.838,"score_display":"85.8","source_stated_rank":53},"run_fingerprint":"c80136f85ff0527c3eec0dbf4e7c25d5510ab9e4aab8f2aca40781ea8112eef5","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":109,"at":"2026-06-17T17:51:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-17T17:51:54.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"13894d58307e459162fbbcb468c7b0ee297a10e8119009f073b83736af760928","metric_details":{"best_score_across_scorers":"0.9333333333333333","model_release_date":"2026-04-17","stderr":3.041173683586219},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"fzoUFmy77NpEDtXyA6w6ap","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"249b023e199255073505beb6a67e06b60015ca533978a6656bff2f43483690e1","result":{"confidence_high":99.29403375316232,"confidence_low":87.37263291350433,"sample_count":45,"score":93.33333333333333,"score_display":"93.3","source_stated_rank":53},"run_fingerprint":"0cd83ed09d3948d59df5c43987b038948a8a63bd1351464f7a2f69540a4e4aa1","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":109,"at":"2026-06-17T17:10:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-17T17:10:48.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"052703e02cd384e347d74cfd5195398343f02c410334f91f20ca1e3b077b6876","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.4280701754385965","leaderboard_private_problem_count":285,"model_release_date":"2026-04-17","public_example_count":10,"stderr":2.9360921879029935},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"bdsZjrG4ayzP8NAJg5q586","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbdsZjrG4ayzP8NAJg5q586.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bdsZjrG4ayzP8NAJg5q586.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"","thinking":null,"upstream_model_id":"grok-4.3_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"b957245226b13d90d28b7cbd6831a61d6ce276763e5442919988d1d8561d271b","result":{"confidence_high":48.561758232149515,"confidence_low":37.05227685556979,"sample_count":285,"score":42.80701754385965,"score_display":"42.8","source_stated_rank":54},"run_fingerprint":"6a44c82d3d4217c53e25d88b5b652bd63ebcb8b2f13e6f49a0c2690d14e0a28f","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":109,"at":"2026-06-17T17:10:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-17T17:10:48.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fab09af97d4db04d34d77102efc0d26eaedf317337bb9d1017b75f1a21a7da13","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.14634146341463414","leaderboard_private_problem_count":41,"model_release_date":"2026-04-17","public_example_count":2,"stderr":5.588506945068097},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"3iRQkf5v7Wu7YuZhxa76Gg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3iRQkf5v7Wu7YuZhxa76Gg.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/3iRQkf5v7Wu7YuZhxa76Gg.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"","thinking":null,"upstream_model_id":"grok-4.3_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"07e2c15de85b808280d356662611df8d1b6699a3abeb5bc117715cef545aed89","result":{"confidence_high":25.58761995379688,"confidence_low":3.6806727291299435,"sample_count":41,"score":14.634146341463413,"score_display":"14.6","source_stated_rank":53},"run_fingerprint":"592ad7f12ca8f9d25ee66af6d070c23cf2d401d721ab4d09f36ac774379ad66d","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8e1fb1bf714af7728efcc498642927e10837fc1a35ddc24c52d265f9c4606924","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.02182,"date_is_proxy":true,"latency_seconds":150.948,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.902},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":null,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0c170d8cabeb6f978d02dbfd8f8571d984558f6645f3f34a6c6177a99f18072c","result":{"confidence_high":84.83192,"confidence_low":81.29607999999999,"sample_count":null,"score":83.064,"score_display":"83.1","source_stated_rank":34},"run_fingerprint":"1c26e395a2cf8f07c0bacd10c72ddd2db526cae2c368f12b18dc47b0985f0961","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_01923756939b60f21f059b03","provider_config":{"source_id":"vals-mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"01c9b7ddb13e7ff2aab5c07ea2f84f495a6e88e222091c81c5d6e046c01b4ef9","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (medium) · mmmuPro"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-medium","upstream_model_label":"Grok 4.3 (medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"3d9ff776c8c2ec7e53b5e2f4c9013f15fc0c5495eab17826e044bdb468e138ba","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.83815,"score_display":"75.8","source_stated_rank":null},"run_fingerprint":"08760be6484c33f047e8ab097de0282fc105593c1c2ba137c492f146fde482f0","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_1c8c583aa239a9f12d2052a4","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"66cf4afd3b93e9d4cf87c8c45efec64875626301610e288b677580b00ba8e7d3","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (low) · mmmuPro"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-low","upstream_model_label":"Grok 4.3 (low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c3914c6e62db5f02d87f20e2ad3b78a0a34ec4b2de0d080fa79a8e09ac93433e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.774566,"score_display":"72.8","source_stated_rank":null},"run_fingerprint":"54a047f80d218bb2a99e66ff9d5e279711fd63ecd6d6c742ed15461989df9020","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_7f815d75f75f06091a49b306","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"41a9c8a89944f39a964a6a8801948f4b38b9c2398907eab064751af9423a362f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · mmmuPro"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-non-reasoning","upstream_model_label":"Grok 4.3 (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4437772aeceb05a039650b34c4e52f5c7ce41350114c1f11154e06f9f0386cdd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.797688,"score_display":"64.8","source_stated_rank":null},"run_fingerprint":"6f342be7d809bb21441b782d1bd323590b82f92e15ff780d4438830cd40237ed","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e036d76e88b5b36c0f50805dec363d478dd6e2ca5fc6494cf546a0c652816563","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-17","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.3 (high)","source_effort":null,"source_model_version":"grok-4.3_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"d4001ed86ec3bdc0d1c637f630622ab52a2628848592aea4ef5838b0d72062a7","result":{"confidence_high":16.669402,"confidence_low":3.642296,"sample_count":71,"score":8.0,"score_display":"8.0","source_stated_rank":101},"run_fingerprint":"27991cb1f4c3510722538dd5467d8fe69ef3631e65e45ce8079097dcd42d1f41","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d8dcfc98e3a496104f7ff7b7161b4e019691f9e352160ecba895ec28d4786c25","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-17","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.3 (Non-reasoning)","source_effort":null,"source_model_version":"grok-4.3_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"4ad9cb99293f8ed6149d7f9bc1621c801110e6cc33c5aa1bcfbb7d2d8d5245c6","result":{"confidence_high":5.132974,"confidence_low":0.0,"sample_count":71,"score":0.0,"score_display":"0.0","source_stated_rank":167},"run_fingerprint":"6848445580515b269a18ab3006ae57276bb7a1905f3d373d9b1322e9bfd65cff","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f1275ffdca3d76d29a9604914740f6fc03a3754ee0c52a4878f288e11903cc6d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-17","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.3 (Non-reasoning)","source_effort":null,"source_model_version":"grok-4.3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"fed0fa7936b475b25eff2a0dc81178ee1d49bfa0a4781560e79cd90720d88e95","result":{"confidence_high":5.132974,"confidence_low":0.0,"sample_count":71,"score":0.0,"score_display":"0.0","source_stated_rank":167},"run_fingerprint":"c2a2cf6407161d4553fe58700537025d0958d2d42e7cec8bfed5e81933e5236d","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3be766358ee2c534df19b81b","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1e9efcdc4afe0165c8fff6c7c2f34c6a177610543acfec7e5d58f4ab291823f3","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (medium) · critpt"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-medium","upstream_model_label":"Grok 4.3 (medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"b5731fac1577d16c756dea9d7613546d29fe86c0a52604490546bc8043589f9c","result":{"confidence_high":12.642381,"confidence_low":1.769008,"sample_count":70,"score":4.857143,"score_display":"4.9","source_stated_rank":null},"run_fingerprint":"2beb41b39e3af9624209e101409b5177459220d5ed64162d9d6b5213ebe86b98","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_1c8c583aa239a9f12d2052a4","provider_config":{"source_label":"medium"},"provider_mode_key":"medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ba4b3ef1a978f10c300fd8fe44797ada4a73ce546b537b80ef026b0c4c410a95","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (low) · critpt"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-low","upstream_model_label":"Grok 4.3 (low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"104479eff5caa9c834775f28443194b84f12c528de58f9c0caf621bdfcf993b5","result":{"confidence_high":6.236252,"confidence_low":0.049636,"sample_count":70,"score":0.571429,"score_display":"0.6","source_stated_rank":null},"run_fingerprint":"b2e9b15bc0ae344202e52ad0d9b5173530edca3153ef4cb42ef6cf777a812e5e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_7f815d75f75f06091a49b306","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a5dd3e723bfa24850bc3983bee613fa868845191394a672d4b9005b72951574d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · critpt"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-non-reasoning","upstream_model_label":"Grok 4.3 (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c0fed610d76fb05fbb1c6b360278b89a183baa2b499891ac4352dafa684e1275","result":{"confidence_high":5.202487,"confidence_low":0.0,"sample_count":70,"score":0.0,"score_display":"0.0","source_stated_rank":null},"run_fingerprint":"6b8dfb7e822295d30b11fa74c14f0c512db69f38ad416fe8fd015a45405b38c1","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":109,"at":"2026-06-17T16:27:46.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-17T16:27:46.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"32435b4845d3dfd112614dcd554bf3f60b4dcb125a7af2bbaad076339c22fa1b","metric_details":{"best_score_across_scorers":"0.25","model_release_date":"2026-04-17","stderr":4.351941398892446},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"R9XpD2PaYpPiokjjSWnMYT","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6f7a084f137a8253927aa6cdedd7c4f3abadb8e086f57a8c2f9e23beb5ac6016","result":{"confidence_high":33.52980514182919,"confidence_low":16.470194858170807,"sample_count":100,"score":25.0,"score_display":"25.0","source_stated_rank":66},"run_fingerprint":"1f9dd751fa63f05b9726e90d26840ae89ba4b0d3d299080a71cbeec2b2f7f641","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"97b45ca9b1f444819ac2b6b29ac51fc9deed3d530729797e4ef6663badf630fc","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.22259,"date_is_proxy":true,"latency_seconds":117.767,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.023},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":null,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.3"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"71bb58789ef2b068e60f12766235e4c0861ec519d44d540dde58a7b69132e33b","result":{"confidence_high":75.36508,"confidence_low":67.43492,"sample_count":500,"score":71.4,"score_display":"71.4","source_stated_rank":57},"run_fingerprint":"271744c8c035d9be49fde8ff9aa1323959280794046958ddc3f845d485cea21a","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"09dd2c3dec735028e5417f7863d0cefadace0cf6dcf1a1edb100f7fb73a8039e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.020464,"date_is_proxy":true,"latency_seconds":3248.962,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.024},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":null,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"4fd591a489e97ef64a6e3ac659806182938fed842548f95156257e27f0a4d427","result":{"confidence_high":86.50104,"confidence_low":82.48696,"sample_count":null,"score":84.494,"score_display":"84.5","source_stated_rank":39},"run_fingerprint":"1f7a17774426b58101a6cdcd137ed4354cb05b1e0c438a9f4dabb0e842283db8","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3c4a8de2dade9c971bb9252924c0045e2655f599abc4debcbeb064c709e743b5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-17","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.3 (high)","source_effort":null,"source_model_version":"grok-4.3_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"a27a22741d24ca301a0936096ac69f7b043fe40cb5b8acfa79b1130b993d4b64","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.337962962963,"score_display":"47.3","source_stated_rank":107},"run_fingerprint":"a8e8ab2c563b1b97b0e9b18349dc61f535de3fde07cffab75cbbc0a7c6b72583","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"545ab8a18c9af4fac1a55da49a2a9c44ba5efe13782cba14cc7712512520b57c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · scicode"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-non-reasoning","upstream_model_label":"Grok 4.3 (Non-reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"4f9a358705c85263e247ae49a03a4f181c0d57f9dd5d6117cba7e3377cae2cd0","result":{"confidence_high":45.098746,"confidence_low":33.885288,"sample_count":288,"score":39.351852,"score_display":"39.4","source_stated_rank":null},"run_fingerprint":"03042f6f21c6eb20bd79e5b76830b559706edffe495e07f12f35dd47efa1eebf","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a8bffdb7c6262d05ca5774c480212ad4653d40364379b65b43f5ba093d384d05","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.055799","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-17","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"grok-4.3","source_effort":null,"source_model_version":"grok-4.3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"5c52cef2c13349fbe526e4444028f3f8d339f3aa4a2cef822aa8893adcb1d572","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":944.17,"score_display":"944.17","source_stated_rank":53},"run_fingerprint":"1d633aa218c41e0678e2ac235fcf019934d2425984487c2d270c51b22ff8a12f","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_43f75a9097da6dcf80b67be2","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0bdb12defd57f88bed4b1148748d1ad23f0bb6390299e7c8851e6e48edaf94ee","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.28472,"date_is_proxy":true,"latency_seconds":589.407,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.243},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":null,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.3"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"205e08af91e5042588c2da713702809173b6b7fb29f220cd01aabba03a95ba0f","result":{"confidence_high":27.719279999999998,"confidence_low":11.086719999999998,"sample_count":null,"score":19.403,"score_display":"19.4","source_stated_rank":78},"run_fingerprint":"b18bf6ec82861277524f63c07d6c83b7326c519c66096942e492003e20f3b057","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"95c5a21d80b433f977e88d3ea5fb964d36bc31fd49a9d72201b50969cd2b456e","metric_details":{"license":"Proprietary","variance":10.074065565386606},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"grok-4.3","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f0ed7afa825bb72cbce24a0bbd00cd0e97494e98b62a531355642bfb098981bd","result":{"confidence_high":1362.846772036272,"confidence_low":1350.4050506077226,"sample_count":15057,"score":1356.6259113219971,"score_display":"1357","source_stated_rank":105},"run_fingerprint":"928badf9bfbc0a72327e23f3bd1ac7233a385829895ffb2c1819f8e3c6f8d5db","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_88f0aaf5c213ac8fa693d6b6","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3bb2f496c9564b521e630842e05788be4b96a0e090653194fe467af909362005","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-17","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4.3_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"a17501db949acfa2519f74516a546538c29fcdd69205303e071872fb426f4352","result":{"confidence_high":1363.56,"confidence_low":1350.78,"sample_count":14192,"score":1357.17,"score_display":"1357.17","source_stated_rank":103},"run_fingerprint":"344e07c66e924cc7992c2ca3fe59bc9f5025632af8db0c9700b5b5f32df4d3be","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2c0cacfc824745ed71a761c4","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"93bff76030423b6a4e47d15849fcd260e43c85bed61e431752f3649f0767927f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-17","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4.3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"6f35a6ba42aae1245d3d9ffefa92a51ab123b07abc7377cca91a179998927e5e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.89,"score_display":"49.9","source_stated_rank":70},"run_fingerprint":"63c155b92115765e7a5a16be488e54e9417403ac697acf6a5eaaec0510cf0aa8","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7774caf4dfcc1695a2cae560","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8f4585745b1b74acc52ed214a757d9fefb7fbc30634fe67c691a0f5f7950fdbe","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.982203,"date_is_proxy":true,"latency_seconds":470.713,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.622},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":null,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.3"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"0662e5fb8546830d960bf3a206bf229550a0b4171a8ea11e19a5d362092d4bcd","result":{"confidence_high":47.08712,"confidence_low":36.80888,"sample_count":null,"score":41.948,"score_display":"41.9","source_stated_rank":67},"run_fingerprint":"c5cd01513a0694020f52b48478ad2a7fed6b4cd3daf80728344df7c567312e5a","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dd9a748a761e1b50aed270c751ec8244397f8a852b16cd211116757b4dda5b63","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · terminalbenchV21"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-non-reasoning","upstream_model_label":"Grok 4.3 (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"c33360e4e4ed991564ac96f2d0740724076ba2978c6b3c8acc1c5f9d275c79de","result":{"confidence_high":44.405137,"confidence_low":25.076934,"sample_count":89,"score":34.082397,"score_display":"34.1","source_stated_rank":null},"run_fingerprint":"d6ef56bd600ee711bf09bbfc1b6e1a4ac405c1ca182917a0ab6dc1df5c0cdb94","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"64838441fa180abe99805a12992072527c71d89b8f267b5f7e8c34864ba30c38","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-30","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · tauBanking"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"grok-4-3-non-reasoning","upstream_model_label":"Grok 4.3 (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"5979f397674c22e4d08500584a4477e8b0036be8b47e06f193d5f083b6fe908e","result":{"confidence_high":15.182695,"confidence_low":4.09665,"sample_count":97,"score":8.041237,"score_display":"8.0","source_stated_rank":null},"run_fingerprint":"27b6d9605ca3c250fede56c8ed7068d55f18ac8c0b8e14dc149b117c01941d07","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"4abd2f6114e6b010340b7aed1f880b04e2739459ad08551eda023a56454c5dd1","metric_details":{"expected_trials_per_mode":261,"normalized_gain":24.4,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":18.8},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":50.900000000000006,"confidence_low":32.5,"sample_count":87,"score":41.7,"score_display":"41.7","source_stated_rank":17},"run_fingerprint":"57f60ac425091578f669e9f453311ae1d584fd6086bd3432b344bab18c4ab785","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_943719f03cd41fdd64848292","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"aef1ff127f712b29345cccb63ed804b601e2676032046fae03a76ea0ac54203c","metric_details":{"expected_trials_per_mode":261,"normalized_gain":24.4,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":18.8},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":30.200000000000003,"confidence_low":15.4,"sample_count":87,"score":22.8,"score_display":"22.8","source_stated_rank":18},"run_fingerprint":"6c0b1d7225317331f8c65a4a7c03ba87ecc02acf3e2e13d070c69744c1e2fdda","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_943719f03cd41fdd64848292","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4bfd60bef50f77fb4bfb2e9c55e3e760f34bc6ec2dc49e3b37ac3b8b67e005d6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-17","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.3","source_effort":null,"source_model_version":"grok-4.3_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"d34d1ddb40f3254c5f4a21bb94521538b32d1bcefbbe4a019858898e629aef75","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.26166666666665,"score_display":"35.26","source_stated_rank":59},"run_fingerprint":"e3f5ff84c1729fa56be8496a469dfa2104029f21bc1da36e018c2eb42b90aa38","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_10dc674a660bf5857afb18fc","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"24c85891305f2b99c6684c9f16a03f7e0a7fbbc20bd357c5770aafa4712560bc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-30","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Grok 4.3 (High) · gdpval"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-3","upstream_model_label":"Grok 4.3 (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"2e69e9f2dc4c0ee94daafea473bda20982dc2fac60588751010a1a6bb5226640","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":923.21,"score_display":"923","source_stated_rank":null},"run_fingerprint":"f4705e72ecb02e118db8604692684cdd9788457a9de27c1e160347c720bfdbf8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a8d922f83837348cb06cd2ce","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"184b99a492898313d7bf92cc53605f5ebd70ec3c447ca086d884b4197d0c3b4e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-04-30","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · gdpval"},"model":{"id":"xai/grok-4-3","name":"Grok 4.3","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-3-non-reasoning","upstream_model_label":"Grok 4.3 (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"9d47e174a9e88bbd3cb1edb949b1ac6cff6ed4bdb3f8c281afdfa90583dc243f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":929.11,"score_display":"929","source_stated_rank":null},"run_fingerprint":"31dfae648c17a0a1cc8c87910b4755e7aa6ac3ba70a49206a0583e9cad221b11","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.3 (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_391f84c7bb85d165acbd4de4","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"05c487eb50041330c672155ea6b5c9332edb2289936a1374b126eb7241a9a054","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Max) · intelligenceIndex"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol","upstream_model_label":"GPT-6.1 Sol (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"9bcfff9914c92424dfabc1fcc8169dceacacc2d5afc7a334949e22d1e743de2a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.83326,"score_display":"51.8","source_stated_rank":null},"run_fingerprint":"7101f81bdecc6bbd0fbee0d03c06c0256c0f24d05804021da7850150b340f9e6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"92bb7de0cd14b5aa8d1d8c564bc0a6ddfd24da5d310ae772431d8b4d724a603f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · intelligenceIndex"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-xhigh","upstream_model_label":"GPT-6.1 Sol (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"119b7d4bb5364791f24e825e8f945728a5d2d7a38220c0c6594698ef95fa9556","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.037768,"score_display":"51.0","source_stated_rank":null},"run_fingerprint":"d7af98aecbd8021c277f0bf5a5edb21662f48f2946cbed5565416b004e799b4a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a7c5e1deed5dafac9c319957727618c23fc613a2588507ed4968aed664f3101d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (High) · intelligenceIndex"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-high","upstream_model_label":"GPT-6.1 Sol (High)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"5ae1aa2f225d10d5e9ac571da1cfe1aec2c56cc28bbc02204c63a34fa3e55930","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.237778,"score_display":"50.2","source_stated_rank":null},"run_fingerprint":"f86146d6dd96e05a08ad6ddacb8589f30a3b05744caca2352f835bd934e1dfcf","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (High) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"28aae5e7d0f14ebc3a3e159a620735a7345e635f91be831448498cfe75443c3d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · intelligenceIndex"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-medium","upstream_model_label":"GPT-6.1 Sol (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"d99191ffa6fef4ee8337351f68b96f23aaec9b29b11b144c42d456b22002e7cc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.783327,"score_display":"47.8","source_stated_rank":null},"run_fingerprint":"1bd96ab76770274c08371c2e8a98cd6feff7f7b4eeec80455808d075a4953389","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e1ef03d34a41517e469fa8ee22ea1bc2c183ec141e5c1c3c29d93eb45f6e7e03","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Low) · intelligenceIndex"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-low","upstream_model_label":"GPT-6.1 Sol (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"c112f2142c5aceb6a05f876d3065b2f34ee349ca6ec5ebb0b072e414ba2ddf98","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.083562,"score_display":"42.1","source_stated_rank":null},"run_fingerprint":"2bb721078a6bcae657354b2162a4051c040f64dcd10b73d2cae2145c547eacc0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5078aae5c21b4e31b95c751e987f1f9fdcf54148541d9c491fdc41eaed4649ed","metric_details":{"license":"Proprietary","variance":29.3939122243677},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1493.8386310019223,"confidence_low":1472.5862896354745,"sample_count":3071,"score":1483.2124603186985,"score_display":"1483","source_stated_rank":21},"run_fingerprint":"47fa6bcc0a94e6ecf9b10b90937a8c0c26ca11b9b094ff9e112ce57fb914899a","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0566d0bf8e9e387218fbe6e7ea45e1a0c8ce6d754dac9754c9c6b6e9fea49d9b","metric_details":{"category_scores":{"Agentic Coding":54.54533333333333,"Coding":80.35849999999999,"Data Analysis":82.65933333333334,"IF":74.15425,"Language":90.12599999999999,"Mathematics":96.83175,"Reasoning":92.6345}},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.61566666666667,"score_display":"81.62","source_stated_rank":6},"run_fingerprint":"8bdb91769e067d271da4b3dab15f69c6c160dac0b039f1ec2f6d4a5d9976815f","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"829e5ecf7736414f988b05008ccd4467b17c5599f4b46573d6f568671e2edf75","metric_details":{"category_scores":{"Agentic Coding":56.76766666666666,"Coding":80.741,"Data Analysis":82.17966666666666,"IF":71.325,"Language":88.645,"Mathematics":96.4785,"Reasoning":91.6345}},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.11019047619048,"score_display":"81.11","source_stated_rank":8},"run_fingerprint":"0399313df14b1171efdfae56c20eb48cd09ec1eba4d52b019060ddc4cc9ea264","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"37480d54c68302a07ef73ceacdc3be43104fff3aa04a5b0e79a7d6956d48c0ad","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Max) · hle"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol","upstream_model_label":"GPT-6.1 Sol (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"cd67934a3abf842c1503b8ac1446ab545c417948a88680fcf72055c6a0394921","result":{"confidence_high":55.018319,"confidence_low":50.810046,"sample_count":2158,"score":52.91937,"score_display":"52.9","source_stated_rank":null},"run_fingerprint":"8ba5491d2f60655f3c21b0facf940eadd330e00e9c17cec1b68677d899cd31d2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c89ce63438564e484c47c92674fa99b61b57b9ad4a9ea16c722b4637037890c3","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · hle"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-xhigh","upstream_model_label":"GPT-6.1 Sol (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"881cecd22579f4bcddaae8c2900efa962d8cdb604822d35d24d815a08be08359","result":{"confidence_high":54.741429,"confidence_low":50.531853,"sample_count":2158,"score":52.641335,"score_display":"52.6","source_stated_rank":null},"run_fingerprint":"9c77aa8a8cc040e206e561a477ca271c84a64aa0c56c539714a64fed9823af5e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7de714f08f6c3674fcae45679c25c9f65be81fa4be7dbd6ac8313ddcf38f42c5","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (High) · hle"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-high","upstream_model_label":"GPT-6.1 Sol (High)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"38a3b1875fe108976beda1459dd64f24ef4b56a4a7bffb3c4a07b03849477588","result":{"confidence_high":53.494618,"confidence_low":49.280793,"sample_count":2158,"score":51.390176,"score_display":"51.4","source_stated_rank":null},"run_fingerprint":"e0af79761646cc43226129f636c8dd09da59c5ab3293e92f23035d7b70c4e593","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (High) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c50910dfa13433a651b045252fb55dbc27b96098ae2ecd417972f04bc40569b5","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · hle"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-medium","upstream_model_label":"GPT-6.1 Sol (Medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b89aa2f44bcbd1fd3943e219f69ae4cd7fc1f5ddc7f936e0c4f226f190ac2653","result":{"confidence_high":51.968947,"confidence_low":47.753511,"sample_count":2158,"score":49.860982,"score_display":"49.9","source_stated_rank":null},"run_fingerprint":"e510ec1fa936d94175ed1ad3c6c23565211a918f1a68234050ecaf8e18a7a7f4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9a83dc037ad5b118012656b9c9ffff5cb8672fdcc083244c70def44808dc1717","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Low) · hle"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-low","upstream_model_label":"GPT-6.1 Sol (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c29f740fda65509456b6fd0bc3ac73a912b518b6316aa27d2eef6c6770439a5a","result":{"confidence_high":49.468147,"confidence_low":45.258571,"sample_count":2158,"score":47.358665,"score_display":"47.4","source_stated_rank":null},"run_fingerprint":"5d89f75e8bf8fc66132a73398d7847dbe02329e0c423cb5586e8885988f45081","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":5,"at":"2026-09-29T22:00:34.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-29T22:00:34.000Z","first_observed_at":"2026-10-01T00:45:05Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6bdf4bb8873da7c66d9079c2a0bf5dfe089e5f26f266334e259d1bd81b2552bc","metric_details":{"best_score_across_scorers":"0.739","model_release_date":"2026-09-29","stderr":1.3895037677965185},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dfT5VD7cQug8XW6GQHyJUF","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"f6b3a1b039acb21492a8c84d82bca5974ab8d86acef1fa62417b47cd67aeb27a","result":{"confidence_high":76.62342738488118,"confidence_low":71.17657261511883,"sample_count":1000,"score":73.9,"score_display":"73.9","source_stated_rank":2},"run_fingerprint":"535eabc5ec536c663454f0179f8552798307779c2439b3a008e7aaf4f964daea","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":5,"at":"2026-09-29T22:00:34.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-29T22:00:34.000Z","first_observed_at":"2026-10-01T00:45:04Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9349b564c1b5153963ccc8b7178399dc93954d14b2a71ecf87f7c296625ee1da","metric_details":{"best_score_across_scorers":"0.9539141414141414","model_release_date":"2026-09-29","stderr":1.3780748450364693},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LspviLnbeZ82WCngWp6rwR","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"5bc1057a9127eb8745ece81bd45ce8e12363f10a506b09c4a0e9d6621ed4742b","result":{"confidence_high":98.09244083768563,"confidence_low":92.69038744514266,"sample_count":null,"score":95.39141414141415,"score_display":"95.4","source_stated_rank":3},"run_fingerprint":"e5c1a64f2c2844a849ee31f0e53bf80f28d433c4cfceab5686ff300aa3185a03","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":5,"at":"2026-09-29T22:00:34.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-29T22:00:34.000Z","first_observed_at":"2026-10-01T00:44:54Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"95594a85ecfecd8da6e4380d2e33b6cc707eea5eb0f7a3601fa574a92124930a","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-09-29","stderr":0.0},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Di7y6sES4PGQbXNkXYhdqM","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5b9da07aaafa7046f2395a09b4048ca4669bbb659b767c8dcc651f9e26ec7a95","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"7dbfd10afb6432131dc6c07c98f648793a4dce941c39fb93327a22c06bc08da6","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":5,"at":"2026-09-29T22:00:34.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-29T22:00:34.000Z","first_observed_at":"2026-10-01T00:44:54Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cb867d5632ad5e618ec615637160904e18c871efc7474e19f66483e9fd532f3d","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.9368421052631579","leaderboard_private_problem_count":285,"model_release_date":"2026-09-29","public_example_count":10,"stderr":1.4434038295343787},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"aBoQLaqnXsLPrsbN8wYgUp","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6.1 Sol (max)","thinking":null,"upstream_model_id":"gpt-6.1-sol_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"702c9408f392e2b8a214bd0c1b07cc3f8e14d785c34bb58e89fff092a08d6449","result":{"confidence_high":96.51328203220318,"confidence_low":90.85513902042841,"sample_count":285,"score":93.6842105263158,"score_display":"93.7","source_stated_rank":1},"run_fingerprint":"9607b1ee12a592c9b62f80dfb69ccba2f07d1c9a6ac863c4705db35f24ebaec9","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:45:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:45:01Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"53e9d02ee35e64b638ace9b4ba1e19f4a55c5bd670a89e3d0bfc1ed04c637ef5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.25374978333333337,"model_release_date":"2026-09-29T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-1-sol"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-1-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-1-sol-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2921570a0438a4d0f49597cd194c8d87d5de98dc53a452863c498b702c27f139","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":94.16666666666667,"score_display":"94","source_stated_rank":3},"run_fingerprint":"a8b05b9aaff7859f1d5b5cf97bacb0b556a630e8ffe24d94f9abf9c02d0d549a","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:45:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:45:01Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0445e517500d159e4184f69377196bad132c68b945c4b6eb5d8db94799c89960","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.17808970000000007,"model_release_date":"2026-09-29T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-1-sol"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-1-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-1-sol-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a2a44defa663b078ead7a21e8220e655457b0d7a48d608e680e97ba65e895e01","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.66666666666666,"score_display":"92","source_stated_rank":10},"run_fingerprint":"57217a76b5aa4165eee703fe089fbb15904cf6b89babc4cf0de4d39afb671528","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:45:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:45:01Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c10f43992f56d3d24fdbd0cdc0df3a5ce835064fcb01421647dda1edc22dcf9b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.13468670000000002,"model_release_date":"2026-09-29T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-1-sol"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-1-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-1-sol-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"dd5a24cf10ef9cf8a8e2263b8ac9987b92816dd3530a555c079edb2ebd29e74d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.66666666666666,"score_display":"92","source_stated_rank":10},"run_fingerprint":"665f905a5add7b00624ed4c64b81a012d5a735344d702dd262a579ab6b390da0","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:45:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:45:01Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"283ef64bdb6cf565f5fec6791f179fcdbe7657cf35de100cede78076d18e480c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.10037736666666668,"model_release_date":"2026-09-29T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-1-sol"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-1-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-1-sol-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9e1fb133d3fd1ba3664e78c9c04bb060f633f8046313c576b01904c493d5e252","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.66666666666667,"score_display":"87","source_stated_rank":25},"run_fingerprint":"60386c9322d8a455354fa86f62c1a10f0a9a6b68a8274b5e8712f99aef883cb1","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:45:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:45:01Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"03625d1d7e5cf9e79242bcfa78ace3b7c66b6a503a02cf26c1898371e0d30f87","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.08553803333333333,"model_release_date":"2026-09-29T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-1-sol"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-1-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-1-sol-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"85dfd40e3afd507dbcb5886c31a65840b3f13bc462448b08ea3dbf3f3cd78530","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.66666666666667,"score_display":"77","source_stated_rank":45},"run_fingerprint":"49420e75d728de77683052635ea0c0dfbdc32ae9900d5323ffe6a93ce57742d3","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ded817b47b75e02ca497984e274b4fdd636a8a40e197986099e6f884e9d2426a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (Max)","source_effort":null,"source_model_version":"gpt-6.1-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"311ae08a7e1b616cbce7ee6014dd735d5bf0c95d6f5c196606ac9ff04b2a9831","result":{"confidence_high":96.153349,"confidence_low":91.247322,"sample_count":360,"score":94.16666666666667,"score_display":"94.2","source_stated_rank":2},"run_fingerprint":"f011f927319c25df94adc90f3fce8e653e6cfc727b29d0f4863b7e0cfa1b256b","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fc289863770927bc073d854d2da3fa6fe210ca65204e727e015f6f3761085039","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (XHigh)","source_effort":null,"source_model_version":"gpt-6.1-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"8b25896873dbcca5acc77850e4b7f83eac237ac65e9d3dcbc55955ed6c15fe97","result":{"confidence_high":94.100582,"confidence_low":88.352881,"sample_count":360,"score":91.66666666666666,"score_display":"91.7","source_stated_rank":9},"run_fingerprint":"ca8d9a366f275367f2413e8b5e8239aa662e5ee9390c435b513127a224fcbca4","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"50358377ec67e72e4d6f94e3205c998169f3f68d24de40ac937d268f061e547c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (High)","source_effort":null,"source_model_version":"gpt-6.1-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5f912808e70fcdca3a9a872e974e97103a0e1b46c07e589a678bac37e4ac9151","result":{"confidence_high":94.100582,"confidence_low":88.352881,"sample_count":360,"score":91.66666666666666,"score_display":"91.7","source_stated_rank":9},"run_fingerprint":"a36d257b2a370d9fbaf5de3c1485c56b87190b32f59d53ddc35bc254348fc854","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7bd6a09abe62f33fdda10d28e55b1264bdd0e93a3220840be39f0859f5d9d681","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (Medium)","source_effort":null,"source_model_version":"gpt-6.1-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"169a3b442c9b51d6ce00da5af6fd463f9393a45d9fbc1d4f76a0fa86f6ea9cab","result":{"confidence_high":89.793886,"confidence_low":82.765161,"sample_count":360,"score":86.66666666666667,"score_display":"86.7","source_stated_rank":25},"run_fingerprint":"54eca8c5d20db8026bbf81115a17c600545c37a64d40e972e77d223a67fdfbc3","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:46Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:46Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2896ba07add48119a0f620cc0a556f5567eede2a673e1d74e975843f93ec9f17","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (Low)","source_effort":null,"source_model_version":"gpt-6.1-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"50ff2c16ccbc34a197e02bfdde5b06535e5b7d88a36e00ed0b01990dff7b2c3a","result":{"confidence_high":80.740238,"confidence_low":72.029979,"sample_count":360,"score":76.66666666666667,"score_display":"76.7","source_stated_rank":46},"run_fingerprint":"d6733ba95b52e1814039931201b54d14f623dc6064d330d21feb298f921d7680","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"32dfa17537b84db8087cb35ca24606ef42af5e330ebc7ffbcdb6f797d5b89b76","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Max) · mmmuPro"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol","upstream_model_label":"GPT-6.1 Sol (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f04786ca84becbfc3ed886e5f68ab301e6811218d0218e3268bf35c47603a721","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.953757,"score_display":"86.0","source_stated_rank":null},"run_fingerprint":"f7457b4dbc06c326607bf91ed8a52e0133371064e993f29cb8af68f4046d685b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Max) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0d2bc0bbb6e5576bb56104336ecf3feeed12bebc5628f8da359d4171f93fa6ed","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · mmmuPro"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-xhigh","upstream_model_label":"GPT-6.1 Sol (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fb8cbca628d23ee725ed000204ecabbb240d9447f3197c33bd95ee4d00036d66","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.086705,"score_display":"85.1","source_stated_rank":null},"run_fingerprint":"fb62aa213322e27e8a50df1456491de9576de73e9be39b69bce16deab87574e4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"deb1eeeb0663b91cf58c397bb042c9497cac2255b02c3acf57e525c79fdbb310","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (High) · mmmuPro"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-high","upstream_model_label":"GPT-6.1 Sol (High)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ad7bd857b44b9cc9d327f7b8b229b31dca8a733f917d1c53a17c5d8058f26cb5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.913295,"score_display":"84.9","source_stated_rank":null},"run_fingerprint":"057479025d644403f003f813d7a764f758462876865c012e30fda32608e95441","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (High) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b06d559afe61af1ad316bbb1a39173bacce7a54b9e3f0c0ecf35cc1bf6fe36df","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · mmmuPro"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-medium","upstream_model_label":"GPT-6.1 Sol (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0126b005a5dfce45b6bb5044dea909084f5dcd841913f25caae16b2e757110fb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.930636,"score_display":"83.9","source_stated_rank":null},"run_fingerprint":"1d72f06c65b6a842d8cfbb2505fb060f294c32d01d3bfcb81d6074a0f47f811a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d87dece3f41bcc7586cdb10446da610c4a9626e82b24a71d67b245c0c88f3a87","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Low) · mmmuPro"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-low","upstream_model_label":"GPT-6.1 Sol (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"bf37638c5d2f22bb9efae83f43a8250f2d359478722d7a9b71b299e46f9d7389","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.121387,"score_display":"83.1","source_stated_rank":null},"run_fingerprint":"39ac1ee939d7b7e84978918c07ba27a111dc54c792219b32583e213b1a96c43e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f1f494050fe1e8f3561015e3a4d1ce215d11dd8018ff190ca21e28b41d8ca6fc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (max)","source_effort":null,"source_model_version":"gpt-6.1-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"efe80311503b042c0e58a17e3e5aff92617f1edbd78b969f7c38a4c48297b141","result":{"confidence_high":43.237892,"confidence_low":22.067881,"sample_count":71,"score":31.7142857142857,"score_display":"31.7","source_stated_rank":2},"run_fingerprint":"14c661fc545b9e5f1c334101df31cbc5afc589a30f517fd2bb9fda519cbecb9d","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"81af640048ce9a6f81e8fc66bb5783f65e0e2aa248b9701f388e842ff3656d8c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (xhigh)","source_effort":null,"source_model_version":"gpt-6.1-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"cf99d8b1faa4c5c388579dd2c9bc3f0f2a53f5e327effaff0b7a65ec08b39ec5","result":{"confidence_high":43.237892,"confidence_low":22.067881,"sample_count":71,"score":31.7142857142857,"score_display":"31.7","source_stated_rank":2},"run_fingerprint":"87cf9663fccfff05593f13138be4f35dd7309d5406feb812604195f76b430c24","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"064401e904b584996b52f04e7d6c6c61348b53b74e45914df84367356344d0fb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (high)","source_effort":null,"source_model_version":"gpt-6.1-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"796d5ba4c9703126e1144eaac2583d755308521ceeb799f9a461d6bbf88a68ea","result":{"confidence_high":41.45954,"confidence_low":20.593649,"sample_count":71,"score":30.0,"score_display":"30.0","source_stated_rank":15},"run_fingerprint":"709e665ed7b433ef85d1313ac05ff57a7c2a1b8a38f0cf1c7cedcb2bce4d0d16","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bd5089c40cefeef631e18688036a79c4828b97209a41fa217034866442024702","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (medium)","source_effort":null,"source_model_version":"gpt-6.1-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5b8ec260630359d43c70448f5a774bab5f5eada4b195d1802ad5a3223d7335ad","result":{"confidence_high":39.063098,"confidence_low":18.653313,"sample_count":71,"score":27.7142857142857,"score_display":"27.7","source_stated_rank":27},"run_fingerprint":"8f977f73800495e4c5fdbea277de684ef52fc779c68a0e253df65f4c7fb24652","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7310102f4d5620486e6cac6086d0715673c71b3265ef3ec9f8be84afc761bd1a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (low)","source_effort":null,"source_model_version":"gpt-6.1-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c9ce58363bd3dc86540f53714c3e789f1a3408af3d1bbcecbd72ee4a861b3f0d","result":{"confidence_high":36.024012,"confidence_low":16.271426,"sample_count":71,"score":24.8571428571429,"score_display":"24.9","source_stated_rank":39},"run_fingerprint":"0f3a4e2cb22e9b0b0fab30807bbc54f4917c5cb2e810c0a3693b39766dff64e8","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ac88abc70c6a094afcab924a647285eb4312cd0530e78f43b185808e2040fb7b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Max) · critpt"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol","upstream_model_label":"GPT-6.1 Sol (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"2d30adc5b46fdbce16b798232d3101f3f206ad0fc36215dbfef1c047289a5b94","result":{"confidence_high":43.322603,"confidence_low":22.008593,"sample_count":70,"score":31.714286,"score_display":"31.7","source_stated_rank":null},"run_fingerprint":"e2c5f9401399eb0db6b4af541b79e933d718276939db6469217b7ff2018c41e2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"284510928ab1b87386e5cb4580675386c2d790eb75e2e26f52fbf2d978ea4e1f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · critpt"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-xhigh","upstream_model_label":"GPT-6.1 Sol (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7e96ec533305c30f0a5596d86c13adeb10216ee5231cab816b93c8722fdff575","result":{"confidence_high":43.322603,"confidence_low":22.008593,"sample_count":70,"score":31.714286,"score_display":"31.7","source_stated_rank":null},"run_fingerprint":"390fc4f95821339b3d3b2fb8e7eb729c114f3112a61e989c5176f8f5efc2fb26","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4beffca687e3074e6b58875cbf59f46df936b5d591c6989d85e71b49a01c8498","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (High) · critpt"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-high","upstream_model_label":"GPT-6.1 Sol (High)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8aafaa485f333e63ecc1c6121a1944b74692cf19859911c01e2b6d8a0c6d9478","result":{"confidence_high":41.544538,"confidence_low":20.536457,"sample_count":70,"score":30.0,"score_display":"30.0","source_stated_rank":null},"run_fingerprint":"3e54764e137f7acca2713c3ffb78bc789a57a34883f07226a443c0635b4fdc44","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (High) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1e1f4839a60ed6d6dc483d04029978d65c23e2f07f49f19862dc63aea7560843","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · critpt"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-medium","upstream_model_label":"GPT-6.1 Sol (Medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"3b2fa126396263c6d8efbe6fbc756df59ff52a1d6cec075e63238aa50b1c97ba","result":{"confidence_high":39.148332,"confidence_low":18.599063,"sample_count":70,"score":27.714286,"score_display":"27.7","source_stated_rank":null},"run_fingerprint":"41fca1465102a7f0928deccee5bf56507e9e2cebce10912a203d0b44dd92ed16","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ab3145d0027acf8b55d412aebcb35c6c33dc82d912f5177b54dd39fb9462afac","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Low) · critpt"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-low","upstream_model_label":"GPT-6.1 Sol (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"0e03565ed4ba6b11ed7270d97f5c768e406f6839448781eb3dc5e9b45081d5a4","result":{"confidence_high":36.10929,"confidence_low":16.221104,"sample_count":70,"score":24.857143,"score_display":"24.9","source_stated_rank":null},"run_fingerprint":"e1cce59666933eb4aa1b13a7b5134c70403c0986b54512fb9bbf5cec7d042885","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":5,"at":"2026-09-29T22:00:34.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-29T22:00:34.000Z","first_observed_at":"2026-10-01T00:44:59Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3fbd658863c05de1a160e33df3ab92e2ba1cde6fda858913a810ca578a0bc223","metric_details":{"best_score_across_scorers":"0.61","model_release_date":"2026-09-29","stderr":4.902071300001973},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"d9TWVVwaH6vgp36FM7qGJ5","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5a58d179163342226493dd4ae3e06d904003728079801ca3521a3e930c88dafa","result":{"confidence_high":70.60805974800387,"confidence_low":51.39194025199613,"sample_count":100,"score":61.0,"score_display":"61.0","source_stated_rank":4},"run_fingerprint":"37acce6678b860060a83f268b157f8d5264288fc73cc0ee1a1ceb3242360d697","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"983729be409be1c28056e2e94eb363539fbe05fb38d2354a00bec282768e7153","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (Max)","source_effort":null,"source_model_version":"gpt-6.1-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5d64108a49bf54a9fc6a3b3849dbe4d14948c397253a818f5ec9b237ebd6a6d3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":96.5,"score_display":"96.5","source_stated_rank":21},"run_fingerprint":"b3ce7cbc33d28df8d20d058d22629732093588ae9188aaf4abc92a7d16aef2d7","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"89349f52e9456cee123b8b5475731321295f038a4169ef4fc489cbdda53cd3c0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (XHigh)","source_effort":null,"source_model_version":"gpt-6.1-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"bdeeebadccd768d1c10bd702e06f8193a86b55e0589fabb30e32412f6216e0ab","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":98.5,"score_display":"98.5","source_stated_rank":1},"run_fingerprint":"d7b668bfd126bcfe46c652a49a776e65fb3ecced9eaa10d990e708b8a9b2b392","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9218b4c27699db5b2c1113229bfc0f0db38e7abf41c7431f4693739a2c2336d7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (High)","source_effort":null,"source_model_version":"gpt-6.1-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"1ebbe793b058e74f4fbc88c32d06ae92a7ebbca038c3e90f936a1f03bbc5ccd4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":98.5,"score_display":"98.5","source_stated_rank":1},"run_fingerprint":"aa708e2c0700a8a9da98ec99bfcfbe2abce376152a8ffcf9566bc3e2cb060c76","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fbed10b207060b8ec4478bcf51149f81ea12325ed5215b28e8700b61fa3b38aa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (Medium)","source_effort":null,"source_model_version":"gpt-6.1-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3e49c1c8e7093803aaa751efdedf5a406226b5a17214400c2f6fdf596fd543ec","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.5,"score_display":"95.5","source_stated_rank":29},"run_fingerprint":"3dcc6f09af579473d79b23cca8811a8c4b3730f619ac8abd0a098948678bcdc7","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05T10:25:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-05T10:25:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d38d7d8b61c05af8cc6e55cf8511949db70396cea3bf57633c57d0a41b9fea72","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (Low)","source_effort":null,"source_model_version":"gpt-6.1-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"c5eb3c8db7fd1baa47fca3edad9d86f9e39a236c8e1337bee49e860d34d7cdbe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":93.5,"score_display":"93.5","source_stated_rank":48},"run_fingerprint":"eb9a8aa2e98bd84cb2e85b5947eff59408e297f64b098670ba232ffb496b0571","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"669ec08f2e3d091eea6204242dd68a069fd44d434d9cdd92b25900002472cbb9","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol","source_effort":"medium","source_model_version":"gpt-6.1-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"codex","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"a162760f44520a96a355010193409303db46962a0d7c9a9d64a0e8c05c69da1e","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":50.23,"score_display":"50.2","source_stated_rank":7},"run_fingerprint":"4e3d430b5d1c46dd17bda35f334e58c989865035ad210c225840ef0410c11e9f","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"46a9c2a83425c3f2b4555efb3010d250e31d22d408fbd93009547ae808573a4b","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.261598,"date_is_proxy":true,"latency_seconds":1890.714,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.111},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6.1-sol"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"0dcd6712514edb6926b5ee958807de876cf011b9d3b032ee2df50afdba24d917","result":{"confidence_high":100.0,"confidence_low":90.79144,"sample_count":null,"score":96.889,"score_display":"96.9","source_stated_rank":3},"run_fingerprint":"8c458543afe2790944bce303b199d65d3069f3cd2eb4795e283e472c684a3f25","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"87d42c91bea51ca96b5aa6d9c6f81ce97efea5781038d7d67343546c782ac5f4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (max)","source_effort":null,"source_model_version":"gpt-6.1-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ef6c50b82dd819301175451837c691506964a5c388ab433e4acb5a0d7562cbcb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.1666666666667,"score_display":"54.2","source_stated_rank":53},"run_fingerprint":"a6756ed1dcaa545335428272d6572f6934bbeb7b1869e4d456097e2bb26dab2f","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cc8b02a290397718e1f49e336730e5ce5d4982e5a325045df8152c662e80f2e0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (xhigh)","source_effort":null,"source_model_version":"gpt-6.1-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"4de7f8722b3948ebdd3517df2a9d92e0ccd97bc1b9abba5a291c090bc65179dc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.671296296296305,"score_display":"55.7","source_stated_rank":37},"run_fingerprint":"729e83f340626ba35201f2e78dfb0f317ee99b580c2ce15da506d8e405aeb60c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2c2e759d6cc85073c34ee32aa6d5da30aeb0fb021d1e78d7c30626da98389732","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (high)","source_effort":null,"source_model_version":"gpt-6.1-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e8ad54a81db28d5c9bd8b14afac3bd68014fdb05e687ecc901e99da77766b713","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.787037037037,"score_display":"55.8","source_stated_rank":36},"run_fingerprint":"59e4727296061b0e3e6f0e348f17f6641b1afd45ae6356f216fc1cb1cbdddc65","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d90d5fe23c5731803a659d6a16b843d4fa247394589e515df97d344a6f874320","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (medium)","source_effort":null,"source_model_version":"gpt-6.1-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"56069d088741ff656761391b5761212b1eff4a890f16a1814c1ad4bc2ff1b91c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.2407407407407,"score_display":"53.2","source_stated_rank":66},"run_fingerprint":"d3ff131208dcddb0c047a82b843949912da8fabf875a723a6a021e30546225ac","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6e9fd82bd1fcd683eabaca5021d2b041c6e8cbdec0032adc968b0b77ce23fc05","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol (low)","source_effort":null,"source_model_version":"gpt-6.1-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"118ccaf78f8466ab60d885ee6b749e1a5d7df5d908b0b4f9445e6db105ef4a26","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.2407407407407,"score_display":"53.2","source_stated_rank":66},"run_fingerprint":"fa28e20276698b76ac9fd12507a20e2efe79d290cc7be80af139c63b4851ce83","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8faeab7c56c372852beb2ece2c83d249a237f4038ceaf32a7c758b226fd63329","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Max) · scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol","upstream_model_label":"GPT-6.1 Sol (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ee04c1e2590697d6c1dede781d9dfb8216dfa23e5e98a7ca6147a0ff2ca2bc55","result":{"confidence_high":59.828702,"confidence_low":48.394938,"sample_count":288,"score":54.166667,"score_display":"54.2","source_stated_rank":null},"run_fingerprint":"f15bebb8f369213a2f88ec422a95865fcff9a82ed7204fb07aea657821a948b5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e8ebd1e9e0434df11f1d1eacae60492a09697c2b1dbd20d366de539cf9a4ba95","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-xhigh","upstream_model_label":"GPT-6.1 Sol (Xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"5f4709fd93a6577958caebb84a5d36146d9386083fc9dcb2c7cc66774a9732f2","result":{"confidence_high":61.296683,"confidence_low":49.896603,"sample_count":288,"score":55.671296,"score_display":"55.7","source_stated_rank":null},"run_fingerprint":"3739d259fa50f5f9103b94a97bc3d8dc54e5397c50b00b559a4e8c57d9c227bf","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d213fb4cec78714fce64b5ff0a53827500c47f4c895b99637dffa7d491b4fa88","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (High) · scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-high","upstream_model_label":"GPT-6.1 Sol (High)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"7ff92571a6530ee7505af9dc0f15f36063c0b525fee512ad9c8a4fa00c29b74e","result":{"confidence_high":61.409389,"confidence_low":50.012332,"sample_count":288,"score":55.787037,"score_display":"55.8","source_stated_rank":null},"run_fingerprint":"08532f4e63e91fda4cb8190a107a7e45b68734e2859d7dafa16228be36f0e73b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (High) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ef5c3667fcd413e9de9724293f90f49c18c9990a22a36e213efc51fb1da230e3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-medium","upstream_model_label":"GPT-6.1 Sol (Medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"8c1a108919e30cb2d066612c126dc8a63f77610a5bdd4295125e06818ee1b8d9","result":{"confidence_high":58.922751,"confidence_low":47.473413,"sample_count":288,"score":53.240741,"score_display":"53.2","source_stated_rank":null},"run_fingerprint":"7616d02fe661e8371039460f9ebaf8f8e39b939f277bac994a95b227677bd05a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ad063efc345c57e6863cb74c654663e2a74c01c462ae6e6c92b9c4f1ee4c8ebb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","source_locator":"Leaderboard models table · GPT-6.1 Sol (Low) · scicode"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-1-sol-low","upstream_model_label":"GPT-6.1 Sol (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"5c1c748b870301624350fbfce0c407947336c2997682acce7735650ea59fc0dd","result":{"confidence_high":58.922751,"confidence_low":47.473413,"sample_count":288,"score":53.240741,"score_display":"53.2","source_stated_rank":null},"run_fingerprint":"c628693d01ff207678c5dde77e211f3e9411f53d27a68d59b2a3c231c539b464","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"65cb8ee894d1b27a86ca881b2a166c6da9db20fa4fe99f9b6c138d45e66e94c2","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":6.234794,"date_is_proxy":true,"latency_seconds":4489.755,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.893},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6.1-sol"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"0979f2b2b8e4c8abb67d3f70c9733243b5875496066225b5a8e5a360b1a016ea","result":{"confidence_high":92.64128,"confidence_low":85.22072,"sample_count":null,"score":88.931,"score_display":"88.9","source_stated_rank":7},"run_fingerprint":"99950d5d22cf27455f10ca7226257409a86183df0f1e98e4a77981941dbd3f91","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4b17d1707ebc67172879e0a9e46ff91c42cda229a2a0a97765fdef6b139960a1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-29","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-6.1-sol_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"4b2ae4548a07c0747e13112d8eab67b43053b3bea59b395b001078db58f0560e","result":{"confidence_high":1777.82,"confidence_low":1739.63,"sample_count":1264,"score":1758.72,"score_display":"1758.72","source_stated_rank":4},"run_fingerprint":"c821520d5e655c404e93a064bed05c84eb2f5240fbe57d0ee9e7c8f8374de4de","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"236b08fd53f81fc44f3414cd1b4034a9119fc38bf8bf57bf1a869c0fb6e06718","metric_details":{"license":"Proprietary","variance":73.76825496740315},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-6.1-sol-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e5213fbc824c710051ca1e1a08495ede977bfd0ddc8e45b2cc5ba910adb7150f","result":{"confidence_high":1774.646407297586,"confidence_low":1740.978754351071,"sample_count":1620,"score":1757.8125808243285,"score_display":"1758","source_stated_rank":4},"run_fingerprint":"659be761133e0395e78e0749561a24a34e7af015cf3e5239b61a09a0b1bf44e3","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1b251bade77c159dcd52e478f7620873dce6676da5110c04814542d1b8e4af98","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.7320000000000001","mean_standard_error":"4.6","model_release_date":"2026-09-29","notes":null,"standard_error_points":560.0,"upstream_source_url":null},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6.1 Sol","source_effort":null,"source_model_version":"gpt-6.1-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"1bb76d336a5652dbc0989df4ea1a7679cfbd5a99503293e37df1d617132b7568","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":60.0,"score_display":"60.0","source_stated_rank":10},"run_fingerprint":"6cdcd4688f9ca2b678b21ccde211763360020a5c88b3df491dae639f95397b72","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bed2bf419491ebbccb30aa3dc06fd8144dfa8315c6f7de015aace70369c40d98","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":6278},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 6.1 Sol (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"be266f30027d59758e347a19635571f1e894a5e6b9c0eb18b4ddc79abc44aa97","result":{"confidence_high":0.1399195691615278,"confidence_low":0.08464406868737293,"sample_count":379372,"score":0.11228181892445037,"score_display":"0.1123","source_stated_rank":5},"run_fingerprint":"7cc9414906bfe6458b6e8da472fcb9dedff329de44efae85b736178026460af2","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2fcfbac70096e6560a7bba1d3468ff0853a6265009898e4bdb6036af3918e884","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","reported_confidence_interval":"-20 / +20","source_locator":"Leaderboard models table · GPT-6.1 Sol (Max) · gdpval"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-1-sol","upstream_model_label":"GPT-6.1 Sol (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"5df3c5204a7772ecf0306d9b20706bd9818b0004285343d11bf23b962ca63bd6","result":{"confidence_high":1595.28,"confidence_low":1554.9,"sample_count":220,"score":1575.09,"score_display":"1575","source_stated_rank":null},"run_fingerprint":"d0085b85e02d54f57f76e640478f85edd6ecfde4cac2c1cc5bc7fc933d141c32","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_4104ef8cd84a9215477511fe","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"78e30516c02159a5d0cc579ca8ecc16514cf9e2712d31efef45d6b07b67a7952","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","reported_confidence_interval":"-18 / +18","source_locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · gdpval"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-1-sol-xhigh","upstream_model_label":"GPT-6.1 Sol (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"7c17636cdfea9b4450c134759bf43646c24074227618a80ca5b9e49eadf4a3b7","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1510.4,"score_display":"1510","source_stated_rank":null},"run_fingerprint":"cdf8bd82933d8575703d33d9ea8cadca78544cd381f9780c87dfd13be96f25e3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_459bcec00071cd59cf8454da","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f0254b04d550f0b1044c3919e943286a4e413ef13b1dcdfe2b6decae513e7a31","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","reported_confidence_interval":"-19 / +19","source_locator":"Leaderboard models table · GPT-6.1 Sol (High) · gdpval"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-1-sol-high","upstream_model_label":"GPT-6.1 Sol (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"135e49ec1c53b36dce0e849d4841d04d1dca70a9f368ba0b30cf16dde1da84ff","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1486.41,"score_display":"1486","source_stated_rank":null},"run_fingerprint":"95f2e1a31dd720a73bb8e68b55117aa015d6f8424f76588b09a90dc2e5516a7e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_f0a02199a412236681497c36","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"707d45a1d29487ab8f2d7311848d5c8aef34495f027d0c7c2f5ae2d0b7807543","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","reported_confidence_interval":"-19 / +19","source_locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · gdpval"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-1-sol-medium","upstream_model_label":"GPT-6.1 Sol (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"e69bdada17ec95e86c921f37c6c4c0e0b86117e607e28f630bb7d670fbd42739","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1432.94,"score_display":"1433","source_stated_rank":null},"run_fingerprint":"4ee8f1b3867fdf351ef347d913b9ff860597f78dde20eb7cd6fc25081aa9a07e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_64133a3ee4324a3d4e8eabda","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6a55174a177aafc930add25002e25831f3bb79ab46a8eda8a300403c2d1b41ca","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-29","reported_confidence_interval":"-19 / +19","source_locator":"Leaderboard models table · GPT-6.1 Sol (Low) · gdpval"},"model":{"id":"openai/gpt-6-1-sol","name":"GPT 6.1 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-1-sol-low","upstream_model_label":"GPT-6.1 Sol (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"48427be6b327949b07ab20f2be9f77f69744f4865692f93a8eda2e0151eeeb2f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1297.0,"score_display":"1297","source_stated_rank":null},"run_fingerprint":"cf27a698fa56f59732a944afc74c2b1e59cd67631c74f4b35c009455abca88ac","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6.1 Sol (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_d897275517ae38822821854a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8884af16b85d9d51576d54f4157ee26f14f49311c521944cbc86dc72a237c18e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-30","source_locator":"Leaderboard models table · Gemini 4 Argon (High) · intelligenceIndex"},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-4-argon","upstream_model_label":"Gemini 4 Argon (High)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"3b5c0f22082e9f1f3c8449d4cf522441addb935c19a2229ec8ae9f1740cc8803","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.560566,"score_display":"52.6","source_stated_rank":null},"run_fingerprint":"b73be17093e46047cb2e00bcb82efe2b76b9de719e020e9ade8ad10fd5f5eccc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 4 Argon (High) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1f729f3d55014f51234e9b8234cd2e91479e93a88b5cbfb5fefbdf51584d7ca9","metric_details":{"license":"Proprietary","variance":20.295792582536265},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1534.0448047381217,"confidence_low":1516.3851952929315,"sample_count":4932,"score":1525.2150000155266,"score_display":"1525","source_stated_rank":1},"run_fingerprint":"cfd3f9bb743548cc700236212c5338d811a0c8cbf2589593e9e44f3b23246461","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"87dcc8425fb29a189474e87108a1cab6e230e3fb385a46a92e9235675cb8961d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-30","source_locator":"Leaderboard models table · Gemini 4 Argon (High) · hle"},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-4-argon","upstream_model_label":"Gemini 4 Argon (High)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a5d2fa050b13ed5b672fcabb03ab2230eb5294841764bf41102456ff0e3f9e2c","result":{"confidence_high":59.163766,"confidence_low":54.990833,"sample_count":2158,"score":57.089898,"score_display":"57.1","source_stated_rank":null},"run_fingerprint":"1a83ca2cec95514a7b3d9963b7b7c2ac3d74ed3b897947874d9a00545e25c096","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 4 Argon (High) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d84ffda956670215f5b1f34fea4944fa624ff31cdd2162d1c168d8ff5d97658f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-30","source_locator":"Leaderboard models table · Gemini 4 Argon (High) · critpt"},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-4-argon","upstream_model_label":"Gemini 4 Argon (High)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"35f4a5e68849d16dd99b5f5a0ada83668749dfbf0ed662498f9cd75c4765cf3f","result":{"confidence_high":38.544533,"confidence_low":18.119461,"sample_count":70,"score":27.142857,"score_display":"27.1","source_stated_rank":null},"run_fingerprint":"e7456e0ecc81a7458efbd79d0e785a13814edeb248a30737f0f874ae6fee5439","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 4 Argon (High) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b4ac4bb6dc1f15a4295932d5612ebecd0031f656064f788d4e1beb2d8c9710d1","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":5.307626,"date_is_proxy":true,"latency_seconds":1459.27,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.0},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":262144,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"high","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-4-argon"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"7433d921a137880e52053dbc750bc4fb5dfdc4362e8a9025bae06e2a90124f2d","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":null,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"57b0dd9dc1cdbacf6d50996c6e362ecaad595464e85ae14d09382ab7837c563b","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b42cf7e74bf1091b9af8a2f705e1e9b60a95b8145b655c91b82c88ae73dbb47b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-30","source_locator":"Leaderboard models table · Gemini 4 Argon (High) · scicode"},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-4-argon","upstream_model_label":"Gemini 4 Argon (High)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"6f32f0e7b24b117a8c4beeca2d57a88064f524b7cff3e9beacfd2fb9c3402030","result":{"confidence_high":67.226698,"confidence_low":56.073613,"sample_count":288,"score":61.805556,"score_display":"61.8","source_stated_rank":null},"run_fingerprint":"5ffc234477338ef46d960200bfb3f5375eca10357a05846e3faaa1c2a741be60","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 4 Argon (High) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"11fcc75ace768a88290e648424c51820251169cd54a2395cb9ddb6c53caa42cc","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":8.10797,"date_is_proxy":true,"latency_seconds":2247.675,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.899},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":262144,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-4-argon"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"40865407a886436ea8ade1bf0350fd8729f9998c2b0a636651cb0ce43fcb388d","result":{"confidence_high":95.62804,"confidence_low":88.18396000000001,"sample_count":null,"score":91.906,"score_display":"91.9","source_stated_rank":2},"run_fingerprint":"9717c57a82cb367ec88f92867ca7ecaefa24e1f9336641b325db539e0cebd645","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5c8a37a38dac3ea72f86bf75b1554b39674fd5514d3000cb016f1bdc36c49045","metric_details":{"license":"Proprietary","variance":45.807556401557925},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-4-argon-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f7a5bbec892039a47dd5d5ba2b31b1ae5b57d517323cc2d49f5a66ef8ade8326","result":{"confidence_high":1692.765709185396,"confidence_low":1666.2351350044094,"sample_count":2422,"score":1679.5004220949027,"score_display":"1680","source_stated_rank":9},"run_fingerprint":"9cc9930c63644eb98abfac50449c5350ac7d1d06f2c03c006e7f6085d2cd79b8","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"274f5f8419c1886b8edeb31514474581d838481aed664b3311c754e9ec8cbba0","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":10030},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Gemini 4 Argon (High)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1ceaebc1c8fd8daf284dca9e0609d0102a719d9879903e491c4b6a30efb51120","result":{"confidence_high":0.09497596084028107,"confidence_low":0.056347909468084184,"sample_count":587353,"score":0.07566193515418262,"score_display":"0.0757","source_stated_rank":10},"run_fingerprint":"04b8de1b26ab75d0c5dcba7f9de5fcf67beef6362eae5237946a581fe13f3e5a","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"24477ef3c5137d8f89f5f1fedb29cf9f4bbcecde592f887368ec48fbd4fdaa37","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-30","reported_confidence_interval":"-17 / +17","source_locator":"Leaderboard models table · Gemini 4 Argon (High) · gdpval"},"model":{"id":"google/gemini-4-argon","name":"Gemini 4 argon","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-4-argon","upstream_model_label":"Gemini 4 Argon (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"ec11b5fd8875d9f344d527f56b46f89dd8a16ef9b95def22df43eeb7de2775d2","result":{"confidence_high":1628.66,"confidence_low":1594.01,"sample_count":220,"score":1611.33,"score_display":"1611","source_stated_rank":null},"run_fingerprint":"04b56b2c092b80de0df074c0bbb6f17a38e29da8374a84402298cbc7649460bc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 4 Argon (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_bdeb2b4867ec518e7c0919e0","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3275be0c074ce2eeea57a24e471d424eaf4f550e78cd680272cac8fd1996c2ed","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-24"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: DeepSeek-V4-Pro; model release: 2026-04-24","source_accessibility":"Open weights (unrestricted)","source_model_name":"DeepSeek-V4-Pro","source_model_release_date":"2026-04-24","source_model_versions":["deepseek-v4-pro_high","deepseek-v4-pro_max","deepseek-v4-pro_none"],"source_reasoning_efforts":["off","high","max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9b13391354c9f06f11a852f8a304824e10318186450dbc0e442c8c585fe32a00","result":{"confidence_high":150.93,"confidence_low":147.44,"sample_count":null,"score":149.13,"score_display":"149.13","source_stated_rank":57},"run_fingerprint":"b271f433ade967f621e79a4cadb6b75025987b4c35cbdd8a619e5376f01579c7","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2f74bd93cdb08957659a5334","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d0900f8279a54f87202a49e3254248040d1a664e9bc35c4cb27291e34703aff3","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · intelligenceIndex"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-pro-0424","upstream_model_label":"DeepSeek V4 Pro (Reasoning, Max Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"a6f0e93859dff106fbd0c6709cf1b10a39877b2ea9dd1ff51a9a88c259f9210a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.86506,"score_display":"30.9","source_stated_rank":null},"run_fingerprint":"9ae1946ddcee0b3d56bace9b81b5eb2a7a36881c91b8f66cbb2d63124a28e0c1","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"30a9ff4719991472b5213d6c534141f06af97b7c55a74eb12b97dcad0c90ecd0","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · intelligenceIndex"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"deepseek-v4-pro-0424-high","upstream_model_label":"DeepSeek V4 Pro 0424 (Reasoning, High Effort)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"7c4319871433937d2d93ee5088c17d36746d77c240099ebf0937580df1101c92","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.105481,"score_display":"30.1","source_stated_rank":null},"run_fingerprint":"7e148d5f246183b622c0b64176d4da27b8674c2a7dedcc06636089afc5ce8311","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7e684f0e3c3844cdf045a80286a9b6f20df71384209e38d86f458fe00aa91e28","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Non-reasoning) · intelligenceIndex"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"deepseek-v4-pro-0424-non-reasoning","upstream_model_label":"DeepSeek V4 Pro 0424 (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"e40b67d94383b3c1bc279cc17aaddc32417592173b26cf8ca5a295f394cf4b21","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":20.834465,"score_display":"20.8","source_stated_rank":null},"run_fingerprint":"177fac660234291a0d11d03a0d66fdde397dc9fae54ccd32f22889d6d7379494","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a27e78d7861d6cc437ac621b0c85db7b85cc5f5234acae01c1710b4361b90e2e","metric_details":{"license":"MIT","variance":4.113718797027405},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1458.9269424578506,"confidence_low":1450.9764251517388,"sample_count":54767,"score":1454.9516838047946,"score_display":"1455","source_stated_rank":72},"run_fingerprint":"b30f96874ba5f823368369864fd078d804ff6f2df9eb291e6de4b4711645cc1a","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fb8735ed829c3d4324ab33281f603c0e7c43ac369108d8da36dc97eebd31190d","metric_details":{"license":"MIT","variance":3.969317467295385},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1461.6658337408714,"confidence_low":1453.8561040152338,"sample_count":57478,"score":1457.7609688780526,"score_display":"1458","source_stated_rank":62},"run_fingerprint":"249b0d9b5bb27a7cc4cb1925d1f7be02a33a4de01b9271f41382498c1ea88b31","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1164185c54b5fe32e70ddec9","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"795de52af1f3771e4951699e0e0523175ac7177853222fd7a14809c4586af271","metric_details":{"category_scores":{"Agentic Coding":42.626,"Coding":69.994,"Data Analysis":74.53766666666667,"IF":62.35,"Language":78.13033333333333,"Mathematics":90.6755,"Reasoning":82.69225}},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":71.57225000000001,"score_display":"71.57","source_stated_rank":53},"run_fingerprint":"c19eb064c375cf57ece1c276b66e6e14c598d2ef0ebfc8eadc33feaa9d151e73","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_192f474c0dccc604e621ccf3","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_1f4bfa5486eb4deec4942449","split_or_window":"provider-report-no-tools","unit":"percent","version":"HLE original"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":164,"at":"2026-04-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"stale"},"measurement_id":"6092406147b9796ef0065ae304dd37ef4fedaed10752880958e00c544e23de4c","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table · HLE (wo / w tools)"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"DeepSeek model-card HLE comparison; tool status reported; exact dataset revision and judge not restated","reasoning_effort":"reported","result_published_at":"2026-04-24","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9d79f00315c23bc47d2293c9003ea42de6ca857854c04113ff97996ca36b3878","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.7,"score_display":"37.7","source_stated_rank":null},"run_fingerprint":"b4dd9bd2014479d5ebb7c2aebb554c42f342c22bb4d5ff2ba7cc81440ae450b6","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table · HLE (wo / w tools)","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_79ec886ae22fa1df9c9fd7c2","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"89001e9a40e5aba3c48a876157cbbed085645b666fbbd3bc66986097a89ab33f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":null,"num_parameters":1598839674782,"pull_request":110,"result_file_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/resolve/refs%2Fpr%2F110/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro"}},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.7,"score_display":"37.7","source_stated_rank":28},"run_fingerprint":"d9504aa6d363d35ef484c216e9e0ee9dde0db81d50507af2dbbadfef57ef1739","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_12f6f292dc9e520e66a0bf0c","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":164,"at":"2026-04-24","basis":"evaluation_at","evaluated_at":"2026-04-24","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"696d70cb88c3ce82fd9993889bd0e87ef2b3ae218df8341be87a70ce0b512bbd","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"official","model_url":"https://evals.report/models/deepseek-v4-pro","source_model":"DeepSeek 4 Pro"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"faa657351e8329600a689b17c5e40d680128f560a12fba46bb4f8056eafd36b3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.4,"score_display":"32.4","source_stated_rank":23},"run_fingerprint":"285111ce6cfe97723d0b8bd2abf159a1e1afd9b074c7337bdf2426f420f2e536","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9dc9b8363888f98544c02eef","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-official","verified_status":"evals-report-official"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5dc5214f4902e17be96d77ff","split_or_window":"provider-report-with-tools","unit":"percent","version":"HLE original"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":164,"at":"2026-04-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"stale"},"measurement_id":"1bb1634b6c42e9a7f4de3453e0398b820912e28d9690daf6324b1a6e4e8aeb45","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table · HLE (wo / w tools)"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"DeepSeek model-card HLE comparison; tool status reported; exact dataset revision and judge not restated","reasoning_effort":"reported","result_published_at":"2026-04-24","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"tools enabled (unspecified)"},"run_count":null,"scaffold":null,"tools_allowed":"tools enabled (unspecified)"},"protocol_fingerprint":"c1a05fd6d83a79a2bdd392afd643d80767c938453c8d68285a5d0cd4f529ee18","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.2,"score_display":"48.2","source_stated_rank":null},"run_fingerprint":"5eb08cc470e85882d5ac171bc7307a98004dda808e64d42a7861e28605e3e2fe","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table · HLE (wo / w tools)","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_79ec886ae22fa1df9c9fd7c2","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"18a733e252e864015834b1cdc57651f87ea4eddd5bd1091751105a61ddc2364d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · hle"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-pro-0424","upstream_model_label":"DeepSeek V4 Pro (Reasoning, Max Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a174714c0c02664b3978f4ef91c041cb4ccea658046754a3e785e15abc4b8e91","result":{"confidence_high":39.5982,"confidence_low":35.51561,"sample_count":2158,"score":37.534754,"score_display":"37.5","source_stated_rank":null},"run_fingerprint":"4e434f129c7bab7dd1ca8b5bd1679b936f5c34d5ec7514bf0c3117b0d1a2cea4","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"06ab7b4a842a2012a437ae8baa12a80486aa5e4c0a7ff9d764be16a6c5cf4a33","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · hle"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"deepseek-v4-pro-0424-high","upstream_model_label":"DeepSeek V4 Pro 0424 (Reasoning, High Effort)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"498d67a064c99f070c7869ebcea98657878d2bbfd0398abc44f2d975c95ae102","result":{"confidence_high":37.25774,"confidence_low":33.230384,"sample_count":2158,"score":35.217794,"score_display":"35.2","source_stated_rank":null},"run_fingerprint":"80a682f7b395a8d3557b3f415ef5ddcc6c06eb2fe1911903486e09e2d8302c3c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3a9ec07a9dda2f7ec8013cc9dd886bf37e1df2ea18dbea63c526d4057297484e","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Non-reasoning) · hle"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"deepseek-v4-pro-0424-non-reasoning","upstream_model_label":"DeepSeek V4 Pro 0424 (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"3eaa787671d44e8075a5be26dad9bbb5e1916a11838cd5fd4e4ddb4afe2de0bd","result":{"confidence_high":9.484614,"confidence_low":7.160528,"sample_count":2158,"score":8.248378,"score_display":"8.2","source_stated_rank":null},"run_fingerprint":"e2b8b545c2d83c778ed9c39ce3ab43c398f61ff9a78e791ba04d3cbb3116c5b1","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:30:29.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:30:29.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"307cbc2b4d20db1346cade0da743a67685b66a530be406f6badc321cf958cfe2","metric_details":{"best_score_across_scorers":"0.46993987975951906","model_release_date":"2026-04-24","stderr":1.5806515332355318},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Sq9xqkVTmFvELpQy9KH5TM","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSq9xqkVTmFvELpQy9KH5TM.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Sq9xqkVTmFvELpQy9KH5TM.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e6b53f7246e0f1d6038e6ea750f2d839e2dc7c2c8caaff3af680b5977c371cec","result":{"confidence_high":50.09206498109354,"confidence_low":43.89591097081026,"sample_count":1000,"score":46.9939879759519,"score_display":"47.0","source_stated_rank":34},"run_fingerprint":"731815a90b3b3994181ae698a5f661c81de7cff06440a0526aa72a616dc282f4","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4166df21830455024d6f8eca2624e80ded8addc8e87aac9fb97084f2ae839872","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.125847,"date_is_proxy":true,"latency_seconds":900.378,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.654},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":90.404,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":256000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"max","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro","zero_shot_accuracy":88.384},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"555e2084ba1b12c35c2f87cc01d6bbbd94274b8965fccbadedbf6a46deaaf85a","result":{"confidence_high":92.63584,"confidence_low":86.15216000000001,"sample_count":396,"score":89.394,"score_display":"89.39","source_stated_rank":30},"run_fingerprint":"4ac9a3347b3500497602772c4bc00feaa08a7ed29ca0f6d53eabcb3daf74000f","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c4c2a1b4da38720c69f58d4a3793a4716a610f553621871ffeb22304b6598995","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · gpqa"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-pro-0424","upstream_model_label":"DeepSeek V4 Pro (Reasoning, Max Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fd39a43496518767e6dd74d6e963b7205d0e312cdb974952b74fcccadadc2330","result":{"confidence_high":92.464626,"confidence_low":83.634653,"sample_count":198,"score":88.787879,"score_display":"88.8","source_stated_rank":null},"run_fingerprint":"5da88b00172dbc16ca82492d4855aad8598f842b734499ce04d2b60e109c4ef0","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3b65226c88fe1c46e8a1b3803c47ce284cba9718d71c2e2844e3c87712f8e8fb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · gpqa"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"deepseek-v4-pro-0424-high","upstream_model_label":"DeepSeek V4 Pro 0424 (Reasoning, High Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c3e35252c36c838fcf381e95e15851d6ece6960d32363359ce3dd7ea94daddb3","result":{"confidence_high":93.851159,"confidence_low":85.617098,"sample_count":198,"score":90.505051,"score_display":"90.5","source_stated_rank":null},"run_fingerprint":"131a6f90153424a5fa319c49143c4997e2797aa5f29b5971393b56caf42c6419","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"319db5f76f76cd05084e65804ee4eb2232200f4ed989f448515a8e77ba862aa1","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Non-reasoning) · gpqa"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"deepseek-v4-pro-0424-non-reasoning","upstream_model_label":"DeepSeek V4 Pro 0424 (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5941af6d8de54521938b689f24deff087bd3350231e93685839eddd9b03d929a","result":{"confidence_high":77.530888,"confidence_low":65.076781,"sample_count":198,"score":71.717172,"score_display":"71.7","source_stated_rank":null},"run_fingerprint":"9a80c93712f7b1fe085ab084a7645b482b24bd5552fd23d9843e348f3de45613","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":110,"at":"2026-06-16T20:16:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-16T20:16:55.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"6af414f6bc1ee24be202709f54decb6e8be547cb76fb28768e4955ef53417823","metric_details":{"best_score_across_scorers":"0.8964646464646465","model_release_date":"2026-04-24","stderr":1.7473014662966007},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LuZUnhD5rBLjRw36DaqfvZ","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"a550bd7755b0e3d10e7ac1eae7fc5980c13c4b59785ff15e92de1afdf7dbe952","result":{"confidence_high":93.07117552040599,"confidence_low":86.22175377252331,"sample_count":null,"score":89.64646464646465,"score_display":"89.6","source_stated_rank":50},"run_fingerprint":"2600aec4ba8ab8849ef5e07b124e9704aff000e48c2b79c64a7eadb246811a61","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T23:58:27.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:58:27.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"eab99d10eda39e37bab8e58acb3d19b198198ed42f4b12473a663993159afbdb","metric_details":{"best_score_across_scorers":"0.9090909090909091","model_release_date":"2026-04-24","stderr":2.0482086775424224},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"TPdRwHTJtaJYSu2Cxc5Czn","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTPdRwHTJtaJYSu2Cxc5Czn.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/TPdRwHTJtaJYSu2Cxc5Czn.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"f63a13a289ebaef284f6246552858cfa6b4dc0a2afbc03a3f43ce1c602f1ca87","result":{"confidence_high":94.92357991707405,"confidence_low":86.89460190110776,"sample_count":null,"score":90.9090909090909,"score_display":"90.9","source_stated_rank":34},"run_fingerprint":"22df84d84ca941d09cac4836afe902017e036f200b1f75005f2cb9cb952fe8b0","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T23:58:34.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:58:34.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"93cbb508ec7a39e21615a538b096c938910f695a9bf64ef41b8105f188024695","metric_details":{"best_score_across_scorers":"0.7323232323232324","model_release_date":"2026-04-24","stderr":3.1544498882702823},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FtGueFbnFnKBAuLmEEufFg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFtGueFbnFnKBAuLmEEufFg.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/FtGueFbnFnKBAuLmEEufFg.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"f4d8e724a48afd850d97b775c08908e3c571fa6f2e726efd1c818b42a5766492","result":{"confidence_high":79.41504501333299,"confidence_low":67.04960145131349,"sample_count":null,"score":73.23232323232324,"score_display":"73.2","source_stated_rank":167},"run_fingerprint":"d57681f4428bc5244877b3e921b1808b4c4f2942b4ea350f0771f1a665351a8d","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7c4b280298f8cf81cd4e3fb80a158a5aa97a209351265da88663ec58c240c16f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.027032,"date_is_proxy":true,"latency_seconds":577.617,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.335},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":256000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"max","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1be5401af20308a986d0b6628a7e8de69fb7700474701af72980e6fce50667ed","result":{"confidence_high":87.90559999999999,"confidence_low":86.5924,"sample_count":null,"score":87.249,"score_display":"87.2","source_stated_rank":32},"run_fingerprint":"45e04b019198938bfc88854c5f0f3b0391b4000a8607525d68aee886a65966c8","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":110,"at":"2026-06-17T00:34:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-17T00:34:52.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fc08d49aeb12fdbc0055c74ce7e62f0b1eacd19fc6d9a508bdd3e3d6c6f03c17","metric_details":{"best_score_across_scorers":"0.9666666666666667","model_release_date":"2026-04-24","stderr":2.002208376727653},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"HjTDENSpy9sancDzr7pccY","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"de87d7ac7927a21c72e918946bc37f3a2db2ead14e8fdf928a8d6809fe55339e","result":{"confidence_high":100.0,"confidence_low":92.74233824828048,"sample_count":45,"score":96.66666666666667,"score_display":"96.7","source_stated_rank":33},"run_fingerprint":"9813b91bcbfda214126b972c4713c8bfb7a53553b1d95931c163b730b8683e78","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T23:58:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:58:58.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0b1ec007a4478e1200ae35884dfc0381602162998d558b396530fc8496e0209d","metric_details":{"best_score_across_scorers":"0.9555555555555556","model_release_date":"2026-04-24","stderr":3.1067790907534745},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"GLZkYFDFDENRiHyL68Nq43","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGLZkYFDFDENRiHyL68Nq43.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/GLZkYFDFDENRiHyL68Nq43.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d96917727f0db48737012a86a8238dbf629861b3fa5216981d25062ba271ecfc","result":{"confidence_high":100.0,"confidence_low":89.46626853767874,"sample_count":45,"score":95.55555555555556,"score_display":"95.6","source_stated_rank":38},"run_fingerprint":"a7bb5b45c8cadfbdfef808078b840a79332379c9cfd68210584ad46810d18b6d","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T23:59:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:59:05.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a991102e033335df151161796e842bbeba0e22d8bd286fea86a52f85b4444c6c","metric_details":{"best_score_across_scorers":"0.4666666666666667","model_release_date":"2026-04-24","stderr":7.521014330903548},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7bwsMdahVHrkaiN5x49Dk7","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7bwsMdahVHrkaiN5x49Dk7.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/7bwsMdahVHrkaiN5x49Dk7.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3589e20bbd7a2e1e90ad92dbc8e06c8035214e35f51f52e4c2c3fb2424ea8f14","result":{"confidence_high":61.40785475523762,"confidence_low":31.92547857809571,"sample_count":45,"score":46.666666666666664,"score_display":"46.7","source_stated_rank":193},"run_fingerprint":"9ec2b5b5d4b71144afaecb6254fa033bf806b5ec6af94a4c1e384ae643582bc9","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":110,"at":"2026-06-17T02:30:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-17T02:30:14.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a6d7463100ddde58597389309ef1a8c21e3dfa7d6d2244265ab770de9f4344c9","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.45263157894736844","leaderboard_private_problem_count":285,"model_release_date":"2026-04-24","public_example_count":10,"stderr":2.953609826992209},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"KUar3FaFjzq5SMKrEhtn8e","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKUar3FaFjzq5SMKrEhtn8e.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/KUar3FaFjzq5SMKrEhtn8e.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"DeepSeek v4 Pro (max)","thinking":null,"upstream_model_id":"deepseek-v4-pro_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"95bf319888c76b6e404d381ba4159ef1791db4120bdc0eaeef0f05a4a760f22e","result":{"confidence_high":51.052233155641574,"confidence_low":39.47408263383211,"sample_count":285,"score":45.26315789473684,"score_display":"45.3","source_stated_rank":51},"run_fingerprint":"ff3934f1ae46e19358382918986457e05a2eb7d101e5f836a1e7a2838db18535","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":110,"at":"2026-06-17T02:30:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-17T02:30:14.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3593955c62b2cc2988ab0c516a8360416d9fcb1dddaf47c60b08f393bbc073ab","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.024390243902439025","leaderboard_private_problem_count":41,"model_release_date":"2026-04-24","public_example_count":2,"stderr":2.4390243902439024},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"RAZ6VMfGtQSE8EvUZRRz4Q","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRAZ6VMfGtQSE8EvUZRRz4Q.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/RAZ6VMfGtQSE8EvUZRRz4Q.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"DeepSeek v4 Pro (max)","thinking":null,"upstream_model_id":"deepseek-v4-pro_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"3b144d21098efd3c14d0a87584ce1cda445178f5d3330ba826845d01323ddc6c","result":{"confidence_high":7.2195121951219505,"confidence_low":0.0,"sample_count":41,"score":2.4390243902439024,"score_display":"2.4","source_stated_rank":61},"run_fingerprint":"d9af6909fb0ec4f3dc6c8133c47a0eb032063e7bffe2f58619c90c771405469a","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"94e702b9bded68fabcb7bebde5934778b87d5b29feeb0b1e442ff9a2d8927e29","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4-pro_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"39daef8e0bc5dc380f96c1cf295c8a188fa83634dccf83f93620986e4e46b7ef","result":{"confidence_high":22.583211,"confidence_low":6.944141,"sample_count":71,"score":12.8571428571429,"score_display":"12.9","source_stated_rank":84},"run_fingerprint":"ec13d20d9cf647f8316e9e4da5bf702987ac3ced4697cd625c4e762780694235","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0ee81b830f84029c495a3e05f7a3dc7436194b1a744362aebf2e4b737d79ccc3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro (Reasoning, High Effort)","source_effort":null,"source_model_version":"deepseek-v4-pro_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"2e8203a4b4f00debf4ecbd15dbd74e293d34e0a1c2dea504e372e6d570177642","result":{"confidence_high":19.153356,"confidence_low":4.953024,"sample_count":71,"score":10.0,"score_display":"10.0","source_stated_rank":92},"run_fingerprint":"9fd9117367a81e3878486140d7d4007960261c3a684e4c7ae47434cccb1c9392","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"54be1ce6c011f52b2981c3672ca25216f2f1a5fabd14c472f8194fb1218f029b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · critpt"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-pro-0424","upstream_model_label":"DeepSeek V4 Pro (Reasoning, Max Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"54c0a3462b0354586e63990949d9c5451c781ef12ee6b0a0886548f23f1aaee3","result":{"confidence_high":22.664969,"confidence_low":6.914022,"sample_count":70,"score":12.857143,"score_display":"12.9","source_stated_rank":null},"run_fingerprint":"02f37bee68b6ef7bace6be03e07d46d8ac594841ffa32ead246cfa6bab99bd63","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1bb0da942ea0d1d1cc72dada1c29ec58b4f68f35c2fd3abb69f7962b3e608874","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · critpt"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"deepseek-v4-pro-0424-high","upstream_model_label":"DeepSeek V4 Pro 0424 (Reasoning, High Effort)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"92ea9216d2aeaec03d351babee630af467caf36b6603103555812fd8a1f68ab0","result":{"confidence_high":19.233122,"confidence_low":4.928868,"sample_count":70,"score":10.0,"score_display":"10.0","source_stated_rank":null},"run_fingerprint":"5f836cef4ad474569a78d490897e5bd89bb4f321cce305f03f775b67fd918f1b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6ea063e2d6b1bdc1d34657055dc2fb35c1fc90fc2e0ed791c1aee47972f51865","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Non-reasoning) · critpt"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"deepseek-v4-pro-0424-non-reasoning","upstream_model_label":"DeepSeek V4 Pro 0424 (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"757ada75e832a262ae59f0e9731dd27e2484bde2768ce084083736a687e29e91","result":{"confidence_high":6.724008,"confidence_low":0.10358,"sample_count":70,"score":0.857143,"score_display":"0.9","source_stated_rank":null},"run_fingerprint":"f347d3ddd1c3811862400060e48c8e4ed1a9414e7281853ce8b84655b190e8f6","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":110,"at":"2026-06-16T19:13:40.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-16T19:13:40.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"cd4699527c669e5c87b689ea5e273c185feab57a1b901cf9d5e8cb2c5b733830","metric_details":{"best_score_across_scorers":"0.2","model_release_date":"2026-04-24","stderr":4.020151261036848},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"8UqmhNg7pKbebcsYAZYpsh","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4c344c8af6c7462eb955c13ffca17141813728cb2f66fafefe063e9a5e39e4c4","result":{"confidence_high":27.879496471632223,"confidence_low":12.120503528367777,"sample_count":100,"score":20.0,"score_display":"20.0","source_stated_rank":87},"run_fingerprint":"4389b9caa8d9bee894f395436ec254da7feb621e43f142a4aa46825106419291","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T23:58:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:58:37.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1aac2aef5ae9e6c7fc11ceb2ac672d6ef33508da9d01fdf393fb0a1c3564b477","metric_details":{"best_score_across_scorers":"0.13","model_release_date":"2026-04-24","stderr":3.3799766898963113},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"NUa5ihrxwwWWPDx9sgb83g","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNUa5ihrxwwWWPDx9sgb83g.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/NUa5ihrxwwWWPDx9sgb83g.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"73f5854b50ec010559da2fa60b00de488a2d4ea14443ef75049bc20051d2f314","result":{"confidence_high":19.62475431219677,"confidence_low":6.37524568780323,"sample_count":100,"score":13.0,"score_display":"13.0","source_stated_rank":120},"run_fingerprint":"06cf4291e2d0cbf7f33e6b16b506bfee4ce939ecaa47d6d0463ae5f4600a9308","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T23:58:54.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:58:54.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d8a19f9e7f7530cc0ddc50e22f888be27c991ea2397aae61e27a42fb9132384b","metric_details":{"best_score_across_scorers":"0.07","model_release_date":"2026-04-24","stderr":2.564323999762428},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"2hHzyijzk3jKujCSxd5d2s","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2hHzyijzk3jKujCSxd5d2s.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/2hHzyijzk3jKujCSxd5d2s.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"00a2adead982851a41d402dc8a62ad93fa4cb6ca5b51f395a8bd643b0f7e92cd","result":{"confidence_high":12.026075039534359,"confidence_low":1.9739249604656424,"sample_count":100,"score":7.000000000000001,"score_display":"7.0","source_stated_rank":148},"run_fingerprint":"a7ad73d819030270897b96155ee881a8ae745861f738b9cbc2923b293efa4008","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"89ed6d4845d997e2817bbdc06bbfd8789ac256df6eaa02753935c7fd1cd7fe79","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.436763,"date_is_proxy":true,"latency_seconds":634.611,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.872},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":256000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"6518ea9b15e466f54ea743e11ff1d1185f043a7a814c0c52d206e5885767843a","result":{"confidence_high":81.06912000000001,"confidence_low":73.73088,"sample_count":500,"score":77.4,"score_display":"77.4","source_stated_rank":35},"run_fingerprint":"4b32b2fc8d9c6ae9a70b943bd37d0cfb6153b80b174028026c31e61ffd32ebd2","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":108,"at":"2026-06-18T17:43:54.039Z","basis":"evaluation_started_at","evaluated_at":"2026-06-18T17:43:54.039Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"55cea38849def61e0cd189cf5aa41f1763c7651d00daa69ca8506f744801dec6","metric_details":{"best_score_across_scorers":"0.7763975155279503","model_release_date":"2026-04-24","stderr":1.897828052158487},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"WUk3f44JhxtAJaSxi6gFZR","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWUk3f44JhxtAJaSxi6gFZR.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/WUk3f44JhxtAJaSxi6gFZR.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"45de701657473576c62adb03d49caa482e63405701d3cb169a505af060365801","result":{"confidence_high":81.35949453502566,"confidence_low":73.9200085705644,"sample_count":484,"score":77.63975155279503,"score_display":"77.6","source_stated_rank":6},"run_fingerprint":"ddd6c3360ff1bbcf72b6f824ee44b5d303f4507a0812bc26a306dc8831d518b2","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"2a386f936fb7e907b8d80ef4a5670300edfd682c732d9ce28fc756773e7bf775","metric_details":{"model_release_date":"2026-04-24","pass_at_5":63.96396396396396,"potential_contamination":false,"sem":1.2930360445412},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"DeepSeek-V4 Pro [high]__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"7e4a46183c41dd422edc5787a41225794e91e14e6c1831d24d9ddfe3a73ac2e3","result":{"confidence_high":42.714530827480935,"confidence_low":37.645829532879425,"sample_count":null,"score":40.18018018018018,"score_display":"40.18","source_stated_rank":10},"run_fingerprint":"f70282efdf208348c11971aac7b8650fa97d785918364c8b52db641895467dc0","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"48089981dc59f3219a5def5e0e003fb716641159e16673c18a65d380ed13a1dc","metric_details":{"ci_attempted":452,"ci_half_points":6.333430597228876,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":284,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":154.71238938053096,"mean_cache_tokens":24049100.743362833,"mean_cost_usd":0.24138687892256638,"mean_duration_seconds":2212.510104011062,"mean_input_tokens":24191606.099557523,"mean_output_tokens":105998.91814159292,"median_agent_steps":146.0,"median_cost_usd":0.2232531505000001,"median_duration_seconds":2104.8520015000004,"median_input_tokens":20628260.5,"median_output_tokens":101213.5,"median_output_tokens_to_pass":100505.5,"median_peak_context_tokens":232374.5,"n_attempted":452,"n_passed":284,"n_tasks_attempted":113,"n_tasks_passed_any":100,"pass_at_4_points":88.49557522123894,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_deepseek_v4_pro_max","source_model_version":"deepseek-v4-pro","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"88739783113936c77f12a3e2bfdf8872eca3d657a584a15474bdca234b72bff8","result":{"confidence_high":69.16528900430852,"confidence_low":56.49842780985077,"sample_count":452,"score":62.83185840707964,"score_display":"62.8","source_stated_rank":31},"run_fingerprint":"63044f08c99197f6025bf056ef5ab64da147a305b01361e7b0c1a5e2e3dab604","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0e4df78797c9d937a31e0a7f9b5ccc8351b81e6ed327515d95f6182e057b4175","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro","source_effort":"none","source_model_version":"deepseek-v4-pro_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"b672ef2a03c8dd178ee749c191c14cd37184eebdde01297553288af327cf08c1","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":17.64,"score_display":"17.6","source_stated_rank":35},"run_fingerprint":"be041bd1dcfe9c567be814d98944d590e9f09f727e76d2f2e2974cd67ae25d57","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"01a0ff731cec26b8d5c31268cc2021d323b51b6198bb4982a524bc36189ff411","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.109669,"date_is_proxy":true,"latency_seconds":715.504,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.953},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"max","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"689a692f38b8969ac015d0b63b8f09d6d6f1fa2017542599ac1d1c48b171efd7","result":{"confidence_high":89.35188,"confidence_low":85.61612,"sample_count":null,"score":87.484,"score_display":"87.5","source_stated_rank":14},"run_fingerprint":"4e9940e806ad4543016a21433042815ec4ab548a594c50c00e3f071886472cd0","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6bc21435f54179270b25f37f039d5d7f4a7317984f376ae2a5bf66d55cd9eb41","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4-pro_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0746c2a76e3dea8e9a4ba6d60699960731fdae218c6d5df46786a9a55f76fb04","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.0,"score_display":"50.0","source_stated_rank":91},"run_fingerprint":"d93b671015c684fca36f91b940cca8fed7b5e8581fd150880407e20153bd2dbc","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c7147f47505590809c335564dc08b4879c82d049fa3692f0e4b9295c1d8e67b4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Pro (Reasoning, High Effort)","source_effort":null,"source_model_version":"deepseek-v4-pro_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1b39fea0a7a24305d1687882d51d6a094c3d109f5007c378019df8a192c56cbe","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.412037037037,"score_display":"46.4","source_stated_rank":118},"run_fingerprint":"b2306627aea85b67fce71308c8cd3eea9ebd29c03ceaa6a6fd5eb9279f52a151","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-06T00:23:09Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T00:23:09Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a79b84b140759005be79ac14de3b9d42419d5cc0bd9b5c0e9b6ac0f20fc00c19","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · scicode"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-pro-0424","upstream_model_label":"DeepSeek V4 Pro (Reasoning, Max Effort)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"371b8bbc384ff884776dcb8b4dc5b5d43c34cf4f0e603cbda5af3f9e724d8f5c","result":{"confidence_high":56.535349,"confidence_low":45.063691,"sample_count":288,"score":50.810185,"score_display":"50.8","source_stated_rank":null},"run_fingerprint":"a031316639be764d3080b7cf300e6a6aef08dc0ae1e6836b4db13dcc2badb461","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9886e5dd2844a16101aad2b917ce0ecaa63e0fa374af7b9be1de86b3a49b4fa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.148062","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"deepseek-v4-pro-high","source_effort":null,"source_model_version":"deepseek-v4-pro_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"d65099c81612af988ffb7a0b93a5544a9a82eb2b87403f6854871af4995567ad","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1006.08,"score_display":"1006.08","source_stated_rank":49},"run_fingerprint":"8e0930623e18ebace2abd883b5bcba2b955957aeb65d06383eefa286a5675d9e","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ad96bf560513b71e2cb184cd73475b61ca3226b06b7160d7ff89a680a927ac6b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.0166","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"deepseek-v4-pro-no-thinking","source_effort":null,"source_model_version":"deepseek-v4-pro_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"52c9c91912e551e28c7755ca19b26a0ca68276973fe36b520970cdee608c166f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":521.67,"score_display":"521.67","source_stated_rank":96},"run_fingerprint":"46bb3e4f512e75d3f1c8aca4402d7de6789e9aacb21e9351a4c00aa5ef9de97f","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"44a8b3ebbbc2fad9ad4719125ec85130508271cbea786d5e4df1786409c0a22a","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.205991,"date_is_proxy":true,"latency_seconds":3418.92,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.767},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":256000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"6d39177693e9b3cb18697b14ad9d9462f150d6fa8fe9e659847757b4ed8d301e","result":{"confidence_high":59.274319999999996,"confidence_low":40.58768,"sample_count":null,"score":49.931,"score_display":"49.9","source_stated_rank":51},"run_fingerprint":"bc9ced4ba17a119030537c6691a6dbe86f6d70c7ab600cf98988ab96aa31117f","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"45f186e068836cd2a0f31ca96d00b9931e33b36aebc35f63c85427fc237a4d58","metric_details":{"license":"MIT","variance":10.335193218680793},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"deepseek-v4-pro-high-preview","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"61a769155ca405547425bba095c316dfb6554cf85febf4d4ef59c72f8d61b172","result":{"confidence_high":1469.7083099434528,"confidence_low":1457.106370544538,"sample_count":13229,"score":1463.4073402439953,"score_display":"1463","source_stated_rank":64},"run_fingerprint":"f969188de88ec45c6a55dc7f870bf0f349a52bc178a87cc4798337840ad69428","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"022b8146618f52b951331fd6e79f35ab61356b2570b2377c149562b3d653f9ed","metric_details":{"license":"MIT","variance":10.361399575700725},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"deepseek-v4-pro","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"81c2f54f42d181bf53e6dcc82b1bd63b5976d7d7e0525ec1314c7bbe4762f29c","result":{"confidence_high":1452.2808880370944,"confidence_low":1439.662981745667,"sample_count":14042,"score":1445.9719348913807,"score_display":"1446","source_stated_rank":68},"run_fingerprint":"e2d352f14c8860f16028b535884b292cec2f9e39e1210d0eb7b680016f5d2e28","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0fae59f84f03afdeda691af2","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"630c887e817cc0e52a1cbfcf77ae19eda17eb9dc1229b0d96b1ffeb6a1399744","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-pro_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"70b47a7b231b2148d5e3104867ba4bd65b928563e769bd0a1a7d662acfeec9db","result":{"confidence_high":1470.1,"confidence_low":1457.12,"sample_count":12405,"score":1463.61,"score_display":"1463.61","source_stated_rank":63},"run_fingerprint":"1615aed514469e825d7f5346b409dc0820b8347da451ff6e127ccdde6cacb02b","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6518d96006b456b0fc73ab99e123a334753a09b4e58daf71f21008cd551945e3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-pro_none","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"343962cdc88f449cb0b525aba0af9308bf6e2a38c14f90847005e54731989dfa","result":{"confidence_high":1452.22,"confidence_low":1439.22,"sample_count":13183,"score":1445.72,"score_display":"1445.72","source_stated_rank":69},"run_fingerprint":"2be342c25d121f7562c8d5bc573e9cdf2fa0f786417a61716193a7b369dfd8ce","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_aa69f796458792e2a6955243","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"82e6d946d1dea467437bbfa4e2a107ae03839a87d4d6e5abe9cfc196cb12871a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-pro_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"3c606a7cea8057175231587777cc7faaccd9bd14896321e77c8caa694fd3b985","result":{"confidence_high":1473.78,"confidence_low":1453.48,"sample_count":3956,"score":1463.63,"score_display":"1463.63","source_stated_rank":62},"run_fingerprint":"3c554e80bb232a214f411f60c35d0205c1a7d32a37ba2d069da4ba54ddc667d3","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_497597fb19cae73e730eef2f","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fa36a86971f3f2b4399f0d16c480f47f2f3b9d32af1aa5013edef130a22a0770","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-pro_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"db710865a7051263576e4aa65d3c94cd9cd42c3f47757dfe56df0f4fd5301319","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.9,"score_display":"48.9","source_stated_rank":74},"run_fingerprint":"e4c7f17f5cc0322f82d0352a72541d80a2587e1e25411ea051a56ce23adfe607","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"209e898ed834b417550c3e606234c7a4ba9439a38417887b36c6dfbdf6470928","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-pro_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"213374bf2b277ba63802e5b51982a9b25eb60d921f9ecfb346ee386ef11cda36","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.53,"score_display":"46.5","source_stated_rank":82},"run_fingerprint":"b0110c8c509bd3eeb3878b260160258d94da3050f77e010a72c22077eb08bdd4","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0a08f3e3be393eee58fc0c96e323d20d463a7da2774866f32b2f9a4a2bf93f7f","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.239191,"date_is_proxy":true,"latency_seconds":1078.158,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.498},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":false,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"deepseek/deepseek-v4-pro"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"4e345a31b05858ce5fab9b045810de097ee7aa493894e473c96bcf35c99f7821","result":{"confidence_high":53.123079999999995,"confidence_low":47.25092,"sample_count":null,"score":50.187,"score_display":"50.2","source_stated_rank":59},"run_fingerprint":"ed663bd9ac000fa352d006f49a28a1e86438d9503718ad3daa2dc2e989bb045a","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_085955b3c249b60f983db38c","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b4f6147f58de39844ea78e37ae20d278ee4842fa4d34f878162d941c9da7793f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · terminalbenchV21"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-pro-0424","upstream_model_label":"DeepSeek V4 Pro (Reasoning, Max Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"845fba098e67d09d40f2fbdf5dd28ee612cc06196eac3b9bef7f394b6971492e","result":{"confidence_high":73.242366,"confidence_low":53.685219,"sample_count":89,"score":64.044944,"score_display":"64.0","source_stated_rank":null},"run_fingerprint":"7b55aeea1b7ed9b4b181122d4608857d3c90f1ecc349a8d6b308137b71a58b42","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"36283a2d45aedbb03f6790e0b6abca8957ee0ee52702d00c9916315d2deaae3a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, High Effort) · terminalbenchV21"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"high","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-pro-0424-high","upstream_model_label":"DeepSeek V4 Pro (Reasoning, High Effort)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"59f0c80b3a5280e887a2efb927b92356b167bf16eaa3f31bb142cc329c77a8e5","result":{"confidence_high":73.916522,"confidence_low":54.447199,"sample_count":89,"score":64.794007,"score_display":"64.8","source_stated_rank":null},"run_fingerprint":"d9733bdc00e67e0288762670d6a20fb01c70225cb0744bf42da4048830d54b48","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, High Effort) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_83e2a53bddae6e68d7ed56ed","split_or_window":"provider-reported-terminal-bench-2.1","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":66,"at":"2026-07-31","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"fresh"},"measurement_id":"feb64c97326020f36706328b90fb8c376b5bb32feae8931528ea084aebeb1447","metric_details":{"comparison_warning":"Different agent harness from the Terminus 2 run; both measurement series remain stored separately.","comparison_warning_code":"terminal-different-agent-harness","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table and note 1 · Terminal Bench 2.1"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"terminal-different-agent-harness","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"official comparison score; exact effort, sampling and harness not restated for this model","reasoning_effort":"reported","result_published_at":"2026-07-31","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"terminal agent toolset"},"run_count":null,"scaffold":"not restated in comparison row","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"2856f01ce5091a1b0fbf9afd2ac976cbff4761be067a817d79b63afd325fbd4b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.1,"score_display":"72.1","source_stated_rank":null},"run_fingerprint":"55ed407f8cca346a8e715adcb865e11cfa2672f9a4efe409a53ade410bb78197","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table and note 1 · Terminal Bench 2.1","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_79ec886ae22fa1df9c9fd7c2","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":4,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"secondary-re-report","verified_status":"secondary-re-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"37ad71c3cfe54c02ab45cec1f88d79184f5c8f44c7e7a46935726ce22ebba173","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · tauBanking"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-pro-0424","upstream_model_label":"DeepSeek V4 Pro (Reasoning, Max Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"7fd2ad946ee97973206f8f5078309493d26dbf48bc1a656bfe529d9b14c6ecec","result":{"confidence_high":39.846147,"confidence_low":21.876,"sample_count":97,"score":30.103093,"score_display":"30.1","source_stated_rank":null},"run_fingerprint":"a82b480f9072654d60a7ab5212f556a571f4db61cebef3d5eed163a646c6eaa3","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro (Reasoning, Max Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":17,"at":"2026-09-17T14:33:50Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-17T14:33:50Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e31737fa69a6a0ced66dc6a29d99dbea79ccc1c14af61da781b44ce94a202055","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · tauBanking"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"deepseek-v4-pro-0424-high","upstream_model_label":"DeepSeek V4 Pro 0424 (Reasoning, High Effort)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"8ed98de8b91bab9f52e292d5f5a16cb9c193fd350cb9da6e705599cbacd80905","result":{"confidence_high":35.72161,"confidence_low":18.463964,"sample_count":97,"score":26.185567,"score_display":"26.2","source_stated_rank":null},"run_fingerprint":"b5352002a2f1841bdf8a464189c0a1dc7a298e73faf5a669aef39273a7863733","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Reasoning, High Effort) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"6d93ddb784e357cd7fa697985209bd1de059491dc5861e01d620343c9675031b","metric_details":{"expected_trials_per_mode":261,"normalized_gain":31.8,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":23.2},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":58.6,"confidence_low":41.6,"sample_count":87,"score":50.1,"score_display":"50.1","source_stated_rank":13},"run_fingerprint":"4602c43769bc08a8edefac7db918794494f31dc3f20c90be2c2a4a57ef633c97","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1710bb809c3113ebff53f7d3","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"0014b79a7c27be3dd4f241127bee41fabe78e247b06327430409409c51559507","metric_details":{"expected_trials_per_mode":261,"normalized_gain":31.8,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":23.2},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":34.3,"confidence_low":19.5,"sample_count":87,"score":26.9,"score_display":"26.9","source_stated_rank":17},"run_fingerprint":"df3d4aafe8c3b1519cea640da143230ff4e75ebe75d3cbb89cf76989ecdd756d","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1710bb809c3113ebff53f7d3","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"01acd41d5e932f3d14e8a47cc1abb0646d087bc47101cf5196173e08cc004616","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Deepseek V4 Pro","source_effort":null,"source_model_version":"deepseek-v4-pro_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"be87d9ad247a6773fa090746ce641b5caf524039d2caa36fb2af7b17aa7b965b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3284.519999999999,"score_display":"3284.52","source_stated_rank":43},"run_fingerprint":"47f3e56ac7f7e17dc1bff5dcf2dc7fc75dd3ee42166b9e09419e7e012325fe9b","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1408cbe9e98576abb4a0f36e","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d85bcdc888ff481e47f29a083d6f58dc662956a9329317850d0f277b4319ba1e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Max) · gdpval"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"deepseek-v4-pro-0424","upstream_model_label":"DeepSeek V4 Pro 0424 (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"430a212fbcc34150dd2c7d3c104e4ea498b3c40f2ad397396be657f3c43a80cb","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1144.37,"score_display":"1144","source_stated_rank":null},"run_fingerprint":"8830cc2e657ffbe70d91274dcdccf2f5141216b47e0341bc1d27f83a269bb8dc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_27a543e4d8b8c6f237f312c1","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"746665db1e11f4c4ad3d4bd329953ecb417f64fb9e80b42afb70a570749217ad","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-24","reported_confidence_interval":"-28 / +28","source_locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (High) · gdpval"},"model":{"id":"deepseek/deepseek-v4-pro","name":"DeepSeek V4 Pro (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"deepseek-v4-pro-0424-high","upstream_model_label":"DeepSeek V4 Pro 0424 (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"703ac9c669dae5fdfc68f782a4b03ed7bb09dd5f932c2433f4b33e018ef04caa","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1140.82,"score_display":"1141","source_stated_rank":null},"run_fingerprint":"448fd65f3d5ace05df56f218345c3981d6e720f522cecda135c47d06e661e06b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Pro 0424 (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_c5963039dee3d753db6b264c","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bb59f8237f5c0fdb0803c17225a53183a77ee9164db210078da27a40d960c156","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Max) · intelligenceIndex"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol","upstream_model_label":"GPT-6 Sol (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"d776a480acc9fc57420555f4038eb67a8aa369a300d06281e8033275e136179d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.630518,"score_display":"47.6","source_stated_rank":null},"run_fingerprint":"33827db002ab7026d7c88a822319622711fb6008c01c3d42e9836e0084d156b3","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"20a2cecc9143f6b1ce2926a401b17f935b579871c630207a3bef078348dfc36d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · intelligenceIndex"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-xhigh","upstream_model_label":"GPT-6 Sol (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"e216952d41216e13fbddb756490895a5cabf6f15d300c82ed2a77b49aa86dc7c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.244896,"score_display":"44.2","source_stated_rank":null},"run_fingerprint":"0b1099cd7836a0825bd169486247f72a2fce8048fcc3f97322b9c2aa2cbd6e69","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d451f037160a779df038a3a057cc1b04667d291c46d01c1163e33d8de92f6a4f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (High) · intelligenceIndex"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-high","upstream_model_label":"GPT-6 Sol (High)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"e1d4291ba6573539356b2956d2e2970bf4d2935227ecb290576429414995b132","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.400451,"score_display":"42.4","source_stated_rank":null},"run_fingerprint":"96f5a2c648507ea23bf006b7efc1ed6a352050621ed4583db6bae2c2e5167961","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (High) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c88adbccf2441bf3dad1564b1ef48e43000921806e66565e01e890219390fa30","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Medium) · intelligenceIndex"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-medium","upstream_model_label":"GPT-6 Sol (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"3ad9268daedf0e5bbae5fe728c1171269f8ea24ba626c945d7cdb126e984f809","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.811897,"score_display":"39.8","source_stated_rank":null},"run_fingerprint":"e3ad7df15cd19260ee90e808902746082d15c786b72701eb8e757abd192c294c","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9c435bab8e1f288dc70d6b6d30f21e812a8c80b5e057d57c9989533217f055b2","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Low) · intelligenceIndex"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-low","upstream_model_label":"GPT-6 Sol (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"4f51463511f75ea84b6fbea2ce39b12b23bc8e847a87961df82896ea9a4ecc71","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.154876,"score_display":"34.2","source_stated_rank":null},"run_fingerprint":"12b9442c30ddbb6bd2a0cb5405fbc639945301a8dc3e908f218503e7f60b8db3","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2ed2e9d24899f29b0ca9d52021190d66d94ae9085f0bace1a1b67ab6a118911b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · intelligenceIndex"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-non-reasoning","upstream_model_label":"GPT-6 Sol (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"88de265f6ec612125d86fdd9124ec073ab6474f5fb1b2c3eaa6ba55f440c6aeb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":28.512141,"score_display":"28.5","source_stated_rank":null},"run_fingerprint":"70f82e397d2f93fa06cca0f0e6a09c6557fb9d0e1f2f7b9abaa6c3df9706361e","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5547b8817f0b585f0d83223724248989205766ab4f419495eaa2540cdd9ea68b","metric_details":{"license":"Proprietary","variance":12.651293743723748},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1464.1847689523058,"confidence_low":1450.2421115377354,"sample_count":8787,"score":1457.2134402450206,"score_display":"1457","source_stated_rank":64},"run_fingerprint":"79fc7c1c9b151194bc300a773fa27b084ead1fe0d1e30ab400e8372cd018fe1e","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"34342bddc35a92853451831325c298ebdb6a077e4f2f974febb8b0e57c373df7","metric_details":{"category_scores":{"Agentic Coding":52.87866666666667,"Coding":81.767,"Data Analysis":81.18933333333334,"IF":68.57475,"Language":85.29866666666668,"Mathematics":96.356,"Reasoning":88.65375}},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.24545238095239,"score_display":"79.25","source_stated_rank":14},"run_fingerprint":"a6656684b13ca31a68e8be43815b7e1eedec89c93ae844d6b7186a3903366ea4","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3a8cdb0aa8d20a6be6d0dd7ae8fbe0034ed65b6b626635f4ca6210835461ad27","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Max) · hle"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol","upstream_model_label":"GPT-6 Sol (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6dc7bf79d149cc8a5b818bcce561f84eb258ebfac9bbed142a05f737636ef2f2","result":{"confidence_high":50.024337,"confidence_low":45.812546,"sample_count":2158,"score":47.914736,"score_display":"47.9","source_stated_rank":null},"run_fingerprint":"c70db6e58813023b7a184601283a3c6e50b0f18ee3235ccb780c2193945ef9f5","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"95e04c4df3e14c7c6fc8f349daa3ea4da63bbeeff10184729311beeb79bf0f2d","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · hle"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-xhigh","upstream_model_label":"GPT-6 Sol (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"bf7934b22e95b913894e36890cbe620a5b95ace283191fa485e2c3f7f488b40a","result":{"confidence_high":48.401387,"confidence_low":44.197517,"sample_count":2158,"score":46.292864,"score_display":"46.3","source_stated_rank":null},"run_fingerprint":"87bc7f79e8ba2d46051c09cc4c65f38ab6e226f5287bb681c7d9d35452f906ec","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2254dac9a9178442a567ee90b4823b6b2e21e11a85768fe4e93c56d1283981b0","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (High) · hle"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-high","upstream_model_label":"GPT-6 Sol (High)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ee6bf1c671db62557ae264093dca148ee244aa0f1959f05a0b97d7c230d8e399","result":{"confidence_high":46.21848,"confidence_low":42.032278,"sample_count":2158,"score":44.114921,"score_display":"44.1","source_stated_rank":null},"run_fingerprint":"a362bc559aa15f65079f394acd270ce7f3d0c4294f8ef7204ae7b8375ae6a1e6","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (High) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"88247d05da92466764c15f60bdd587d733c14f31772b43781464220fc06fa374","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Medium) · hle"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-medium","upstream_model_label":"GPT-6 Sol (Medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f47d26f039e3255d8350e8bc8a72a245d9a38fb5320f93a2cf1a24929a5a8f2b","result":{"confidence_high":43.052995,"confidence_low":38.90683,"sample_count":2158,"score":40.963855,"score_display":"41.0","source_stated_rank":null},"run_fingerprint":"90f84aa0343966d6212d19f085c4dce34e52aed0ef92fe9e96a2b60dc62224ff","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cde3b9a10a4eb28bb6085451bf67352d4f91f27963f9c55e090a1ce269ae2c1f","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Low) · hle"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-low","upstream_model_label":"GPT-6 Sol (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5c92b9bd426f3b4a867722b526040fe2ac025ff259e28970b56a726a17d1e813","result":{"confidence_high":36.976541,"confidence_low":32.956501,"sample_count":2158,"score":34.939759,"score_display":"34.9","source_stated_rank":null},"run_fingerprint":"2218b6ff90723705ce9c06dc419a7dd0d5527cc94756a7a2170212c7f509dc87","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"65c04bb60b2d9dbe8dbe23987d904df39e58e51382afb02f9b5d4839ad3c3983","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · hle"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-non-reasoning","upstream_model_label":"GPT-6 Sol (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"522b1393ef7b47a30e53fe7d59d82fbd67922b030ce59baad60e9997c80f439d","result":{"confidence_high":20.087095,"confidence_low":16.818552,"sample_count":2158,"score":18.396664,"score_display":"18.4","source_stated_rank":null},"run_fingerprint":"400b4cabaf5127bd8581149647fac2c2d2292a46b6f8f7f671afc5b08c57b407","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T19:59:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T19:59:24.000Z","first_observed_at":"2026-10-01T00:45:05Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"390f0930c376f285244bc7c47e6adfd3b8556a7433cb770d81e6d1e58695b61b","metric_details":{"best_score_across_scorers":"0.607","model_release_date":"2026-09-22","stderr":1.5452824654081525},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"donMeEKBZZdqpQsojKWJEm","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"418b64e983ba024c19a2a130b1474da08ea4b1b41b7e78d42b4539f491bb8e98","result":{"confidence_high":63.728753632199975,"confidence_low":57.671246367800016,"sample_count":1000,"score":60.699999999999996,"score_display":"60.7","source_stated_rank":14},"run_fingerprint":"fe16b649f076f26c25fd00f97171d6c50fcc337dfd9d897439c393c14e372f11","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":12,"at":"2026-09-22T19:59:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T19:59:24.000Z","first_observed_at":"2026-10-01T00:45:04Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"17f036938cf3335abebc29cc9c2d2ed072ab18e9e95f9f6502d1452cf60a2d79","metric_details":{"best_score_across_scorers":"0.9431818181818182","model_release_date":"2026-09-22","stderr":1.4938512008159717},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"aJD7ZfFftu2cZoToJhhzF7","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"a810ed65ab93af76f7fa38055d2734f05182535f56d8816c76257cbeeceff2e3","result":{"confidence_high":97.24613017178113,"confidence_low":91.39023346458252,"sample_count":null,"score":94.31818181818183,"score_display":"94.3","source_stated_rank":8},"run_fingerprint":"6a792c91d911bb8bc65b45728794e8a4e05ef431d9edde6c81b79203c5b64148","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T19:59:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T19:59:24.000Z","first_observed_at":"2026-10-01T00:44:54Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"41e150761b5d2681e718f2ed661e6f0cad0cbfd199bd2b3f000da11a3b267368","metric_details":{"best_score_across_scorers":"1.0","model_release_date":"2026-09-22","stderr":0.0},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"VZAUcNkzeMSMm9paxBhj6B","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c54492e9d4068e094b3ae279760a6322c0ae91c104cd7424160602b3e18c3af1","result":{"confidence_high":100.0,"confidence_low":100.0,"sample_count":45,"score":100.0,"score_display":"100.0","source_stated_rank":1},"run_fingerprint":"afaa47a050a0535fe5087d51f02fcd519946af97b6cf9bd63112f7462ce15464","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T19:59:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T19:59:24.000Z","first_observed_at":"2026-10-01T00:44:54Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a819ce4148dbdfdd207eaaca4b994b36f9e1483ea9aeede232fabcd891e059d4","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.8982456140350877","leaderboard_private_problem_count":285,"model_release_date":"2026-09-22","public_example_count":10,"stderr":1.7939690721286368},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"RHjHEPHEsWZqFuMjBTeC6r","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Sol (max)","thinking":null,"upstream_model_id":"gpt-6-sol_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"c79b1fe7255a430d7782a3c82799bd0574be029d6b4d8fd89c716a787ac624d3","result":{"confidence_high":93.3407407848809,"confidence_low":86.30838202213664,"sample_count":285,"score":89.82456140350877,"score_display":"89.8","source_stated_rank":5},"run_fingerprint":"62ef0cb3b091ab069cb97222265ebd709fb9055b20bf5db0d47b931a915d1bb2","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"275c3d605e5390ecdb6e45736536af60cc143b386f2ef4eeff0f5af49135c586","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.43889828333333336,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-sol"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-sol-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b4e761fc2548ad008906036cdcdeed912c2582cf7697a517cd71e289a02236d0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":89.58333333333334,"score_display":"90","source_stated_rank":17},"run_fingerprint":"5d91cb0e1ed3946d94df4068961fa81a1833825f102bfe6dd130465fbe03f9e5","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9e407a882b678433ff6c931fc6031d34c5a17f0d47a4a65456bc1a482b80b275","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.27722933333333344,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-sol"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-sol-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c6bb7c0ec0cf77fed252f02d961da74fc6fc16be846d4290838f11442d0177cb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.05555555555556,"score_display":"78","source_stated_rank":40},"run_fingerprint":"54573f6e5e1684ca50f6319e962a8fb57d1939bfdbebe0beea374cf7188bf870","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"20dd457ad65bdc2cca5cb5e49e9e6cd7d79060a032e88f55785e14bf4e69504d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.20273668333333333,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-sol"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-sol-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fcd8ca057090f5451ca6517d96137aff22e4be2c7cb0dc160b5f5a4d84b1e353","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.88888888888887,"score_display":"69","source_stated_rank":56},"run_fingerprint":"79f15fe5ba299c19e72db61d668d254fa476cd1b08b8c3c00ceddb1ec1235d2a","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e8ab6437cb246c728fa0f8ada081f88908e0bbc05243a162ccc446057316920e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.14762708333333335,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-sol"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-sol-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"12be176c3837a8a65b37bf8747adb6336466714743ca036f867f9f1ada606bd7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.777777777777786,"score_display":"58","source_stated_rank":84},"run_fingerprint":"c5a5d03a54f19cb829d084a7094ebaf96e2222c694215a7e88ddff0b2ce2d47d","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"171abd69bf8c73c9884b16a6d8b993a69bf4d8aa627b97737d4f995a380447fb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.1008582166666667,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-sol"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-sol-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"01e1d3a7d8eef885582213e39ca2ee275d0a248b9e42796f611349281cd664ce","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.52777777777777,"score_display":"32","source_stated_rank":115},"run_fingerprint":"1c05494572b4fe3407bdac8fd295dcba0881ca6604d73944ced963e1a251301d","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fa509b66fcfa9803dc3635d965b4904391f5c6c5b959721ba00a5d0903ef56b0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0740642,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-sol"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-sol","model_type":"CoT","upstream_model_id":"openai-gpt-6-sol-none"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"36b427800cc4c7ffa076b178a9042bfcbf5ef1dc1e18d0eb60b40f4f68da003c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.6666666666666667,"score_display":"2","source_stated_rank":207},"run_fingerprint":"4942f7843b82ecf6e588907ef95caabe49a4a7da22722c0ebbc39ed94415dbfb","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9e6d8b1041c765fbb2a194556a9e5931dbe59b9bca1f62a78c544637fb95094","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (Max)","source_effort":null,"source_model_version":"gpt-6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"075566d24b8afd6d5b15dc689f762e3cd17afd01756b5f0d668e62814247c5a3","result":{"confidence_high":92.332,"confidence_low":85.998791,"sample_count":360,"score":89.58333333333334,"score_display":"89.6","source_stated_rank":17},"run_fingerprint":"d15e25d454fda58722a31b146be584e4d581d38d116b3621435df1081babe49d","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8e129d0e31f826d79dc641b06240c622c52870cc7739e406093d94012dadb5e8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (XHigh)","source_effort":null,"source_model_version":"gpt-6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6f9022aa608fea7a03019bb35561b89d7989aca8af1abe41f6eb6de07160e55d","result":{"confidence_high":82.022331,"confidence_low":73.496335,"sample_count":360,"score":78.05555555555556,"score_display":"78.1","source_stated_rank":42},"run_fingerprint":"d1604f92092b5d8147d2ceca5adf326e2b2798e48399637cc6af4c18e14efe3d","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9a7b2ec3b7294ba0c608b0875d31f30b36b13501ecede3e949874776250e0b0c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (High)","source_effort":null,"source_model_version":"gpt-6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"5a97d4c825119f796ad218d8d5aabc547b7cb1bb71b03a196272035867b1cb12","result":{"confidence_high":73.450621,"confidence_low":63.928283,"sample_count":360,"score":68.88888888888887,"score_display":"68.9","source_stated_rank":59},"run_fingerprint":"6d1e67aaff85bb4c9f9d638c172594363cab619e4650df822ff75ec0e3131eb1","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"624d92ea99aa80b88f1f683421bdf8bad67edd4564d438c6e20816b2d66c2c69","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (Medium)","source_effort":null,"source_model_version":"gpt-6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"8bc73acebcb834100cb05cc3d2039ccc40fa50608fb3609733e2d22816bd2127","result":{"confidence_high":62.771494,"confidence_low":52.619819,"sample_count":360,"score":57.777777777777786,"score_display":"57.8","source_stated_rank":86},"run_fingerprint":"845405ac24a0742d037959165c410e5e44cec11ffca857597e1b745f6df1ec54","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ce88a46899fb46553f100ceff79c25d917eede2ee6d7ed43d8b9e98a2227b279","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (Low)","source_effort":null,"source_model_version":"gpt-6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"04868308f8dac6a9ae5d9a7e0f4edc3d7206d564f3af7e35ef62949bcc877bd5","result":{"confidence_high":36.501033,"confidence_low":26.944599,"sample_count":360,"score":31.52777777777777,"score_display":"31.5","source_stated_rank":115},"run_fingerprint":"040daf63b444f16dc01b65c3c1ffe1a1e7523dc36038c3b4b01ded4d586abb69","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"873dd22067a3367e4df12dac91a0dd8d3d6f10e26f0d1471755a306d217af924","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (None)","source_effort":null,"source_model_version":"gpt-6-sol_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ed2a667f618fbf11e676e1e2d88bdfe25432054ff91d507938ecfea40b2c319e","result":{"confidence_high":3.587963,"confidence_low":0.766019,"sample_count":360,"score":1.6666666666666667,"score_display":"1.7","source_stated_rank":201},"run_fingerprint":"147ea67d9c9076d6c6b53612c924073e93735d9cc4b386ae851d3e51699e1bb5","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T19:59:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T19:59:24.000Z","first_observed_at":"2026-10-01T00:44:55Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"51b4df5446d7bb0d0bbc4867ba8de8dc582354fd096ddc0dda1365d89a681bd2","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.9","leaderboard_private_problem_count":41,"model_release_date":"2026-09-22","public_example_count":2,"stderr":5.173952057462543},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"jwS3LYLCPbZpHDzzkX6LyV","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Sol (max)","thinking":null,"upstream_model_id":"gpt-6-sol_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"5eb0d83a930ef1849d72bf9941be0f428443b42ac0a1368b3d6eb934ecde6f2b","result":{"confidence_high":100.0,"confidence_low":79.85905396737341,"sample_count":41,"score":90.0,"score_display":"90.0","source_stated_rank":7},"run_fingerprint":"ba58145691233f69d25e0937def2def27eb77dde43efe7f0e2efdeeffdfa98e0","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e97bc10b5b602a401118f5d5b5a52a256b1c1d47c69dc0be56958933906567b9","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Max) · mmmuPro"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol","upstream_model_label":"GPT-6 Sol (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"79d7e035e0b2097f236b40d1ac9970abf0f6442601c9cce7248e27665d4b93cd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.947977,"score_display":"82.9","source_stated_rank":null},"run_fingerprint":"dd89003585964b02290da255b09e95414f7a8b6b844a49773c7f35ee9a65950c","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Max) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"14bc792addf74a401bc6172488824cf363fbfa43ffc29598575c2dd28deae893","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · mmmuPro"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-xhigh","upstream_model_label":"GPT-6 Sol (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9354033740ee696abc2dd550f619876ce5c7bf4d0e2ebd8ad5929bf6b5dfd59b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.543353,"score_display":"82.5","source_stated_rank":null},"run_fingerprint":"baf56dc04d7d56f3d819b05efa57e46c429ec4e59ae544ddfcffd8dcb829eda9","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"06576099b79962b4edfcb092b86e5c8c926204c8891a1bc3e8914835c9dbd41c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (High) · mmmuPro"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-high","upstream_model_label":"GPT-6 Sol (High)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"054278246748c7322f50ee8ff67f9f84b3b3a1789ea850f57d23d9cd1edc6e87","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.965318,"score_display":"82.0","source_stated_rank":null},"run_fingerprint":"3bd75736dfa5cd54b6400716a179c78ff44dd1f55769cd525424303790b1bc75","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (High) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3db94244cc58baf0e08a261fd6b0fa1a1d612296b2cf06ed3a1486a20715cdf6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Medium) · mmmuPro"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-medium","upstream_model_label":"GPT-6 Sol (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"32c0d1b5d61fa24ac6ffbd2e99b7a5eab3aa0028f9f2d89b1680901ea8406519","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.404624,"score_display":"80.4","source_stated_rank":null},"run_fingerprint":"4a49485d84aed45fb04692850049a56eda951da054389ca282d6f8a08ffd4f2c","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2d63ac02ae0bbfbc7e13f907dae06827061a060e6e74dfc0b2297ebaa97a342a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Low) · mmmuPro"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-low","upstream_model_label":"GPT-6 Sol (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"69956d773205b5dfef21bb7686ec546822521fa17b6c5314b9b7d8b0ea5f08db","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.17341,"score_display":"80.2","source_stated_rank":null},"run_fingerprint":"30fd970eb25d2f2bf8e1d10884427a8487c8bd02b77fbe619aec995f9db3c8c5","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"28c635b246e3524c6a009637e41c551d88da06cf4559234af448d9d7931e0697","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · mmmuPro"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-non-reasoning","upstream_model_label":"GPT-6 Sol (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"cc9c0405f0a651598c911bb4a8b4c7812798e1447862eb0c62957bd79e3f8ba4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.346821,"score_display":"70.3","source_stated_rank":null},"run_fingerprint":"990d22c03eb565562023801ed242dfae5591cda7d8e2305a451074248d697854","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"23a941940c9551d36d2e6db36f36effd522cb410441651476e850f621953309f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (max)","source_effort":null,"source_model_version":"gpt-6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"f2be2b9a553255e29cce2c1a1ed8c134c60465508c29980c3c88029310efa294","result":{"confidence_high":42.350693,"confidence_low":21.328788,"sample_count":71,"score":30.8571428571429,"score_display":"30.9","source_stated_rank":11},"run_fingerprint":"116ae3cad391d04e54083966dbd6416deec77b42c47aa6e79cf36c43e9087c73","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9ca43f8e5603103c391700f9305ff19ec484fbf1afbad3d83dbb2a46722105b3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (xhigh)","source_effort":null,"source_model_version":"gpt-6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6b19b0965c775a2008deccdabc281636313731e81e78f7bb1630507ba55fdeb1","result":{"confidence_high":39.364291,"confidence_low":18.894218,"sample_count":71,"score":28.000000000000004,"score_display":"28.0","source_stated_rank":26},"run_fingerprint":"cf79abc6c544aee866d070f3da3322feb73c0955a7b800360f2aa1a9b39d9ec4","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"54ce342686262e37460a428eb3fc5e873645d899e87eea056f190096f340302c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (high)","source_effort":null,"source_model_version":"gpt-6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"061bc8c331ef675787ceb280609ac502e0d8b93ed3a4d278aa8de91c1403c8fe","result":{"confidence_high":36.635896,"confidence_low":16.743737,"sample_count":71,"score":25.428571428571402,"score_display":"25.4","source_stated_rank":37},"run_fingerprint":"56326245aca28aa92b807b34e1d0c8d8646ed2659d1223b6729ab92f56f881c2","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b9322058a06c2f94e01bbf39d759fa78f72a77a331a6d001d60cde7155af9b5a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (medium)","source_effort":null,"source_model_version":"gpt-6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"dba40a619b34f885f0e24b1cc9113310168b2d4f3506368407fab7f240e457ff","result":{"confidence_high":35.717279,"confidence_low":16.036062,"sample_count":71,"score":24.571428571428598,"score_display":"24.6","source_stated_rank":41},"run_fingerprint":"f8fb92415efd2c2d3a5faef401cabb62c05cfda9e3a03c6303322270820b0330","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ef30620b4b55b02f9aa33a2bea07bbb4e312a70386ff404fd582730895a109ac","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (low)","source_effort":null,"source_model_version":"gpt-6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8e0a4f9e12cd0ccdce26b986fb0b7525e8e50d056572a488377efa14ffb15c8f","result":{"confidence_high":26.558802,"confidence_low":9.473717,"sample_count":71,"score":16.2857142857143,"score_display":"16.3","source_stated_rank":74},"run_fingerprint":"83798ee169862bbb2fbd37a006bd7c371c5d45fb400fb0060c865428c9353557","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"db1ed8b722bad47c0a03bff911c43a823223f0026c46594a6916a1d3bcb9caeb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (Non-reasoning)","source_effort":null,"source_model_version":"gpt-6-sol_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c47a42307f851511c8a9bcc5417389e6e8b665c640fa0c903862e44843da2cb8","result":{"confidence_high":11.389661,"confidence_low":1.332676,"sample_count":71,"score":4.0,"score_display":"4.0","source_stated_rank":114},"run_fingerprint":"86bdf38083edf83a18693334a4d0444facd73476591f92544b4be94b7e4c7da7","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7743239105a0c72bffb4c9d52e52442f4766a412edeaf9cdf152dc2dc45fea4a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Max) · critpt"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol","upstream_model_label":"GPT-6 Sol (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6438fa01ae3793b00885de0da4b08ea81718fe6125e17c22f085d0c75b63fe62","result":{"confidence_high":42.435559,"confidence_low":21.270536,"sample_count":70,"score":30.857143,"score_display":"30.9","source_stated_rank":null},"run_fingerprint":"7e4b9c09d378cdc1a7a28edc16efbec5938d424d13d59676ae4127e035ab72af","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6ac381a600fc793cdc9aea8b28e91de70a099e73b2ff78f975e8d7ce5b003c73","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · critpt"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-xhigh","upstream_model_label":"GPT-6 Sol (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"2aa5481e987065c15beb69a5e8c424349ae2c1d07d7f2862c057bb10ef2b730a","result":{"confidence_high":39.449504,"confidence_low":18.83959,"sample_count":70,"score":28.0,"score_display":"28.0","source_stated_rank":null},"run_fingerprint":"8672136599dbcb60815e7adb27b5c9d3db658e243658d11d40fc386e689e514a","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"779c9732692cc37101586e185e653340219b15285a0e89458b8f4213bdc227cd","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (High) · critpt"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-high","upstream_model_label":"GPT-6 Sol (High)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"be77e819402aba0ce76915c9bc427e776be38e453e24211225d328f12d209bee","result":{"confidence_high":36.721187,"confidence_low":16.692606,"sample_count":70,"score":25.428571,"score_display":"25.4","source_stated_rank":null},"run_fingerprint":"ee498bca378a8d8e90cb5e164ce4ecd3b8ba3e48750898289fa48cacab009605","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (High) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dce68288fcc7aa3497765e60724056bf1e7b3c5bdf9d9713676307359ca2e163","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Medium) · critpt"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-medium","upstream_model_label":"GPT-6 Sol (Medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5c507b116c05277c558a8c24b1fece9ab134488aef6281e57543c809cd706e51","result":{"confidence_high":35.802545,"confidence_low":15.98615,"sample_count":70,"score":24.571429,"score_display":"24.6","source_stated_rank":null},"run_fingerprint":"e92bdf454753a492fde5b080a5f0cce782a7cacf5f49b39218e5e47fb65d5d48","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e79fcbacd92386f330d2a33859cf1d1347206365aec4d3c51f65a52b0d716c15","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Low) · critpt"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-low","upstream_model_label":"GPT-6 Sol (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"bd605453f445361b6254750687bc687ae3962d78b7bd31b70144125f77570982","result":{"confidence_high":26.642277,"confidence_low":9.437114,"sample_count":70,"score":16.285714,"score_display":"16.3","source_stated_rank":null},"run_fingerprint":"228f097c73f29cdd92c43705ab3d9db0ce3dfcd8b076eae4bcfa93e484377a48","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7c1899b8eb81e1aa7d85a6a1ad4f5798103bbfee10b44e25aca37ba1afc8fef5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · critpt"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-non-reasoning","upstream_model_label":"GPT-6 Sol (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7a17c497db5ba8e9814ab0c115f029dad19a3841d68c3274ace844a71f9f110f","result":{"confidence_high":11.463124,"confidence_low":1.323165,"sample_count":70,"score":4.0,"score_display":"4.0","source_stated_rank":null},"run_fingerprint":"96d634c4a52585fabcec9f0d0a8c58d9ef737e9810694ff1c7bde7778e359eb9","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"22af82eb82eb8d8f2219c33d10713d48258690f80fbee39ed2f404e8db50e3e3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (Max)","source_effort":null,"source_model_version":"gpt-6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d7b031deb2993c493ad55c165aa96a6e265dce5585f86616aed811a10d6834a5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.5,"score_display":"95.5","source_stated_rank":29},"run_fingerprint":"ae05da08f7264af82f3fa3834c1afd1e8068b985922a74bffabbc6f4c0cf5769","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9c3ee2bc070d4e6f6ce3ea06b3cb75d4bc89dd5cd4054d6970305f8b2f7d58fd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (XHigh)","source_effort":null,"source_model_version":"gpt-6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"75b20177b23ce882d3ee9474e42be393c2bf639208baf8dce06fd466f0613ee5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":92.66666666666666,"score_display":"92.7","source_stated_rank":53},"run_fingerprint":"4d8ed96b7a87a54ff50f421d1e4ca1cbfea4a50d45bb296c90bb6467ebec558b","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1fdc604f0e474cd4a0b5421d435d58fc68f563af86988ab9d3db60d34e7d4a12","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (High)","source_effort":null,"source_model_version":"gpt-6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"af8ece28c6dc6b2d2165ade0003e6df72573a7efa9d86ab319b9be1d840567df","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.0,"score_display":"91.0","source_stated_rank":67},"run_fingerprint":"4e12fd692762040657a4ff27aa39fcae6e2323bdb13611c0c3b6f0c10894be5a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"824194d0b4395b5ccdc1e0c26f16a96a150280a16245a09c82ce394172c30ae1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (Medium)","source_effort":null,"source_model_version":"gpt-6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"41c04de080c4edd14fbe86da308ecf1b16e83f51fde5a24b77c6b2da52d9265b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.66666666666666,"score_display":"83.7","source_stated_rank":108},"run_fingerprint":"3ae115a2352d770ebaf6c6edacd4d60becdc55a46af55c19c7962c9e78bc37b9","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1ea3e455349f53efe33da71203ba22c792f3bcaef9643c8df61fc4d9b0f97f88","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (Low)","source_effort":null,"source_model_version":"gpt-6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"12ecbbb925715e307bab9b9b1dbf766e06053f3c307231cc23db5201961f675d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.16666666666666,"score_display":"72.2","source_stated_rank":132},"run_fingerprint":"cd1bf283511aa1c0bcebe328bec60b52f838d6712d6b3cf4f5c8bb5a8582120a","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"264d8baee9f413a5de5fd6a7698e88d2d8f87b742e01db22a578af551e71a1d0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (None)","source_effort":null,"source_model_version":"gpt-6-sol_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"44e50a121c2d135e915cf6fc4a6cd6e65d5d09effb232df859d757ea43af7762","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.333333333333332,"score_display":"29.3","source_stated_rank":207},"run_fingerprint":"4108e4768f9e1bf658fca3387466c0ba5b989953577d178763701d00986adb05","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3eb023163a08f9f0f7f0e207ad3f8783840a714ec665173ebb0b11f21ff323fe","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol","source_effort":"max","source_model_version":"gpt-6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"codex","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"37ba1673ea5b1f77ea7bba1408caee27aacdb2cb0784d7417b3a75b89083fd5c","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":49.27,"score_display":"49.3","source_stated_rank":9},"run_fingerprint":"51d1b652d6950ebc073c308dc5f100942035dd32185575516fd1e85286dd82ff","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"23139fa7c03483027eaee195e09ed1e2414a9939945e716eb4ecaf660717a46e","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.703348,"date_is_proxy":true,"latency_seconds":1837.553,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":9.24},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6-sol"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"9085a87546ea09449b76d45d57504bc5e0488ed75bc57f3fa7da843c6fd6f178","result":{"confidence_high":100.0,"confidence_low":64.5006,"sample_count":null,"score":82.611,"score_display":"82.6","source_stated_rank":10},"run_fingerprint":"7f018e3cb976d12c0ffbae18f7c5f033cbd1faabda6f265a2c1869f9e29b7ead","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7eebd4432eedbf94d11a02bf14b424c6b69b960fa766016d8eaf5fecf75e16c6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (max)","source_effort":null,"source_model_version":"gpt-6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"c2cb6c8ddbada4a9ff6552d77f028582a6f66766a75736b8403df8bfb946aaf7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.63888888888889,"score_display":"57.6","source_stated_rank":19},"run_fingerprint":"fd75550277ee5101e9a219bec0f5485282b7ead345a733cb5402efbe6f2f0a69","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"297e9305c96120845f8950cdc6e8e7b8c262cc4848e94a0ee65f86cfa8021f40","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (xhigh)","source_effort":null,"source_model_version":"gpt-6-sol_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"6a3b809545b968b523f8f138c265a6acb8ee4d534a9dcaeb83765b41bf0da144","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.0925925925926,"score_display":"55.1","source_stated_rank":43},"run_fingerprint":"d92339b0f4f1cc9d4cc016935c1d3a602087898f4f4612634da4520aafa4aff0","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fb1b687efd61e3915044d552fefeb2349b32fecbe9dcee4047e9276d69906d7a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (high)","source_effort":null,"source_model_version":"gpt-6-sol_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b0d4c24fdcc210c468798ddf31138ec476539d895ff11584c46cec2b29dafed2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.86111111111111,"score_display":"54.9","source_stated_rank":46},"run_fingerprint":"cc67fa8adc1524040ad24c64bb23e2bacb0621afcacda2914a9dd9ad8b00dcb9","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"46aafd6e83851c47cce9b19980430ea1bd68339de9d9e695f71c6de2f55181de","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (medium)","source_effort":null,"source_model_version":"gpt-6-sol_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"665ec68f4739fe4af61d146b099775d2aa0c0a5379c5f6f8b01e4446cacc4565","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.8194444444444,"score_display":"53.8","source_stated_rank":57},"run_fingerprint":"baefbd707da89a50167d782a9e632d48758420aea192f8273ccc541b394afef1","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6e6f56658d15f8f34784562ef44d0360ae54df5eabd3326603b0c22c4cb403fe","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (low)","source_effort":null,"source_model_version":"gpt-6-sol_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e5ecaa9730b70fe7fc5fd2eb5ab21fd03e3be01c9eadbef1da55c8cf28f9a404","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.2314814814815,"score_display":"50.2","source_stated_rank":87},"run_fingerprint":"daf689ded04d8db8f4f8a8f2a2d57fdc2a7b548ff0fdf633f10e4e81166df5e5","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c5ce7a33c0e6dcc140624ce9209657c6b1a5c5fd84b90a2e57ff755eb1bc494b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol (Non-reasoning)","source_effort":null,"source_model_version":"gpt-6-sol_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"d9d1cc62ceaba569080d2c478a3a8a468cb5dfce4fb8f2937ef93c17fab56fc8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.337962962963,"score_display":"47.3","source_stated_rank":107},"run_fingerprint":"67ef222b2f75f429a12cc4cf135869f728fb26e5fff574628296e993dc04c548","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a1e7638c0d2c4c2fa8ffb73c1ffef615acf4b6d1cc66e7d3e2792c02b27ab778","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Max) · scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol","upstream_model_label":"GPT-6 Sol (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"7c045cdc085fafdfa5dfe2e16fbd0bc9cb1fa9c1742406c90672767ff9d8b401","result":{"confidence_high":63.208456,"confidence_low":51.868216,"sample_count":288,"score":57.638889,"score_display":"57.6","source_stated_rank":null},"run_fingerprint":"258f48197a2d0cac853b7481d4ef6c666a3e96ab65d8c541ec2015b826889f79","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"09f4733fb0d0fc4ca053db87d544ac1c3f592927a4cfbb75cfcaa6f905966e76","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-xhigh","upstream_model_label":"GPT-6 Sol (Xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e6adc162781c2025c45b2d45dc767577b9c94ad156aec9a5bb0b08dfa922a9ea","result":{"confidence_high":60.732691,"confidence_low":49.318424,"sample_count":288,"score":55.092593,"score_display":"55.1","source_stated_rank":null},"run_fingerprint":"ea7e0090ee374abb0540489f9161921d9412d0587037bfb6583ffa1823845378","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"65f91063a5ccaf865cdaf91a0940948e402db6e010a209bd91cb7a902721b845","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (High) · scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-high","upstream_model_label":"GPT-6 Sol (High)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"6eef554acdd08977ba4cff4add9442f269f513c4b9a2d591ac94eca43c54d834","result":{"confidence_high":60.506878,"confidence_low":49.087368,"sample_count":288,"score":54.861111,"score_display":"54.9","source_stated_rank":null},"run_fingerprint":"344a78424f9518140858feb59f20bd2133ef771926881fbe830ed29b120b6e6f","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (High) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"90957fe8f8f02a3577926c71821a0fb2bcbd3f2f1f773d3ed1dab0b426dacf70","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Medium) · scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-medium","upstream_model_label":"GPT-6 Sol (Medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"b6438cd7b0ae4b89b3813014c452f8c2d8f36a3d687f40ded134e5f586d42cfd","result":{"confidence_high":59.489199,"confidence_low":48.049136,"sample_count":288,"score":53.819444,"score_display":"53.8","source_stated_rank":null},"run_fingerprint":"29267bc3b362f81ba00e8e6403e3a1614b93ddf54f90bd2e2b17ceca245a8c80","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2800a02a4534310007a5e9c4224fa7a90b431f9a070d3614d88465c2c8def44d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Low) · scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-low","upstream_model_label":"GPT-6 Sol (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"67ed34e26dbd938532b6bd052cf7eec5df6f94e7b10d259a2ba9626127963232","result":{"confidence_high":55.964946,"confidence_low":44.491922,"sample_count":288,"score":50.231481,"score_display":"50.2","source_stated_rank":null},"run_fingerprint":"a6c9ce60f42487b3ba9023d83dfa6e9cb82ffe222a02ab130c49098acb697121","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-01T09:31:57Z","observed_at":"2026-10-01T09:31:57Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"799dc1756e1bc2cb57fc62a3a0617547e65028d39dda89e552dabf820363c90a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":872000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_listing":{"last_listed_at":"2026-10-01T09:31:57Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · scicode"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"off","source_content_sha256":"3ffc69ec68bdc4d417051326b6445091967f0664569885bc971443ec5c03a932","upstream_model_id":"gpt-6-sol-non-reasoning","upstream_model_label":"GPT-6 Sol (Non-reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"c740f61362252d03cd2281ad50f979c61863d062d4e96e37e2b3da6d4c77b9a1","result":{"confidence_high":53.101548,"confidence_low":41.644461,"sample_count":288,"score":47.337963,"score_display":"47.3","source_stated_rank":null},"run_fingerprint":"f3010559e0b1cb05ffca25718a63537157d20b43a80276dadd96295bfee251a6","source":{"content_hash":"7533ecef7c0995d9a94fdb3780d0dfbfa9e0d54f8e0b362e43a6456dc6d934c3","fetched_at":"2026-10-01T09:31:57Z","first_fetched_at":"2026-10-01T09:31:57Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-01T09:31:57Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:58Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:58Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"84764abe663e594bb61a90c42ecfb05a2ba1ccad24514ca245176b110575d3d6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.843984","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-6-sol-max","source_effort":null,"source_model_version":"gpt-6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"6a34c99c14532b1f26a2789f7735f72f05f0d81749399afa6ba4f8498f756292","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2462.0,"score_display":"2462.00","source_stated_rank":2},"run_fingerprint":"6b87676227fb7033c4deabaf8325ad242faf49b2d3156e34f06c425a2b0982f5","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3befc1b13dd7b83626da3384c008d0c164b167844f55270686eb619cd81fd8be","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":26.356012,"date_is_proxy":true,"latency_seconds":2336.393,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.526},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6-sol"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"49105c57996484b8f6fa67d37ee024d81ea157fdb428241ef5fad045957532b4","result":{"confidence_high":92.77496,"confidence_low":82.87304,"sample_count":null,"score":87.824,"score_display":"87.8","source_stated_rank":9},"run_fingerprint":"b4ad0e64a85d00fc36128b4ee25f9a043ea8fd952a3817cac62ece8f93c24e23","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5201cfab8974a0bc661905dc7eac51ad54154b3c7241463a3367702bd274b500","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-6-sol_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"a9777ab04935c1bd5788f629dbbd5c998f2f6cf4dfee3bddc142fc629331796b","result":{"confidence_high":1701.14,"confidence_low":1677.53,"sample_count":3197,"score":1689.33,"score_display":"1689.33","source_stated_rank":7},"run_fingerprint":"29d903a5b5a93383357a71205159d8f9b7a3269a9c43036f65c77baca7883879","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7818378932955205f32916b4e7f125c052e8f18bd3d3dd721bc0f23ee3a31b8a","metric_details":{"license":"Proprietary","variance":34.15143491043591},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-6-sol-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ade50dba58106c48aeaa612cd95a4379433dfa275484a51a54360e32de210dbc","result":{"confidence_high":1700.4149369315166,"confidence_low":1677.5071800901273,"sample_count":3411,"score":1688.961058510822,"score_display":"1689","source_stated_rank":8},"run_fingerprint":"072de7db432f94295149aea89c4807ba74ca100e6b05b0f5bbefabce942385e6","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"123241e978a9858735b0647a04c574fe3aa16ac6f478a596abcb612f48825b74","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.373024,"date_is_proxy":true,"latency_seconds":298.876,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.297},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6-sol"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"132287b0ead4c7244282c16c0053fe03dd866daed2e2e2b9ddf02170126b5027","result":{"confidence_high":85.68812,"confidence_low":80.60388,"sample_count":null,"score":83.146,"score_display":"83.1","source_stated_rank":6},"run_fingerprint":"a734d150cbc3dd73028ffade4615cc1c6ea694795a24fc1816ea98aa234d7465","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"de005e47e4cb0d929fdc5d8bd65e3ae1e94e6a4455941adf063c0da05aa57b5e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.687","mean_standard_error":"4.7","model_release_date":"2026-09-22","notes":null,"standard_error_points":560.0,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol","source_effort":null,"source_model_version":"gpt-6-sol_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"8925914aa7ddad54b83d99b35c84a19e0c4ddb360e1f4602f3262ae9a48909cf","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":54.300000000000004,"score_display":"54.3","source_stated_rank":18},"run_fingerprint":"dea577c4cd588d12a18a6a2afbca5e0fc536caaf43b4566989eef3315104db44","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"86921ad063930533520a1f76cddc96312110a186ace518c67859baaab967052f","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":6106},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 6 Sol (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"47a2f2960099fa340c1883a7b882b4d4f48682dcc572edb9f010605101c1d690","result":{"confidence_high":0.12077420049599813,"confidence_low":0.07344719892801793,"sample_count":994367,"score":0.09711069971200803,"score_display":"0.0971","source_stated_rank":6},"run_fingerprint":"22eee725944444773f7ee7a0ae8e511d9d0ff86009b398b12425acd12e5f39f9","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:58Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:58Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"50e1fed35b085e62546964be4186e1541531bd63488784f770df885b46069b28","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Sol","source_effort":null,"source_model_version":"gpt-6-sol_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"b9e3ab661cfd42b634c3fa7df2ed91045ec8e64a4189ddc39278315065b437b1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":14427.848333333333,"score_display":"14427.85","source_stated_rank":2},"run_fingerprint":"c47dbee47a963f6e7bb8bbdb13cf46df4f5b82f9b595539ccf8f167705d15220","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_88460deff22c56bc1bda18d5","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bbeb2b2e0e3444aecefd63e49e37d7b8401dfc452b4d4824abc7b44dc7ac0ca0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1050000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · GPT-6 Sol (Max) · gdpval"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-sol","upstream_model_label":"GPT-6 Sol (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"3bf314c803c3fd8c4bd7d5c1312fb70cfa7039189b8c73e401b509d9b897f338","result":{"confidence_high":1530.25,"confidence_low":1480.19,"sample_count":220,"score":1505.22,"score_display":"1505","source_stated_rank":null},"run_fingerprint":"9b46dba3d3f1312450751e32a16e56bb6a22fcf6e5cee3c6b3b84dff518c93d4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_0734f29421da39f59df77231","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5b8d8cbcc1a2df09446bb6fbd39ae28acf20834ac21884ba384e4344073b6699","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1050000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · gdpval"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-sol-xhigh","upstream_model_label":"GPT-6 Sol (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c758ce0a07b580543b476274ca035a2ce9b608b5bf07cf7f97ed4bd3dcb99ef9","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1456.56,"score_display":"1457","source_stated_rank":null},"run_fingerprint":"711a7b85677b4392c39beed6f2dcc6e56dbe27adc38b2fdef43306fdaa8eeb87","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_95ca0426927f6f7637753fdd","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2d6729ae493d9152c6221a285ec2e9165e9c712812b2937998fe4d502e843ecc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1050000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-6 Sol (High) · gdpval"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-sol-high","upstream_model_label":"GPT-6 Sol (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"bd446fbd5289618e3bd4cfa43e1395ce3fb7d09ab5752d092a2079239aed9fc1","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1396.35,"score_display":"1396","source_stated_rank":null},"run_fingerprint":"ce624ed561cc87715b733b83cf97d7c5f334ce56fbb76772d6ad542ad5ea12f3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ab70b0528e6cf453062a2348","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"620fb28f7486d7e0fc21c83889fd587d24aaedfaa361ce85e97467ab80b4459b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1050000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · GPT-6 Sol (Medium) · gdpval"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-sol-medium","upstream_model_label":"GPT-6 Sol (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"054093e13237dc3a7b9a5cca355a8ef3af221677e6bc941f30d2daae0ff7270d","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1347.09,"score_display":"1347","source_stated_rank":null},"run_fingerprint":"2dae8e2cbe1ac2989f708ccaa7a011d7950e1faf36e71d589afc6804b47c7cbe","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_2f4572b0b88bd301c14280fe","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6561790baac87bbf3816af30b89ac67a4b3a5a5948e3c83ef1e943a95f535305","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1050000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-6 Sol (Low) · gdpval"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-sol-low","upstream_model_label":"GPT-6 Sol (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"e675ab5a5a90a68918fabeef0823c43299116431798bf3b80e43b14f53955372","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1206.67,"score_display":"1207","source_stated_rank":null},"run_fingerprint":"d901154c8c98bb82849c13ef5e055a7b5c13c080a1614f11c5103bb6174cd584","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_40024ce614b0682675402e56","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b27db17e7ddf5c74b0ac546878797f45590c37eedbd71828c5cc89fb2dfd3d91","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1050000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · gdpval"},"model":{"id":"openai/gpt-6-sol","name":"GPT 6 Sol","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-sol-non-reasoning","upstream_model_label":"GPT-6 Sol (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"cff80bfa1aa3968acf4a3f7e3466fe8ffc337f9a3c3f8c82c89ed49761ab3415","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1253.71,"score_display":"1254","source_stated_rank":null},"run_fingerprint":"48efd6d73fb880a006e50d69eab6d73d1e470445111d5769ce577249e35b933b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Sol (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2ce50b2fd1347cb6f6c5e37e","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b7f27f6675296e55ec38f4e263656606066d81cfa6a5c8d5a8ea3b49d3a85e09","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-24"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: DeepSeek-V4-Flash; model release: 2026-04-24","source_accessibility":"Open weights (unrestricted)","source_model_name":"DeepSeek-V4-Flash","source_model_release_date":"2026-04-24","source_model_versions":["deepseek-v4-flash_max","deepseek-v4-flash_unknown"],"source_reasoning_efforts":["max"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6d4cbee18f0b0c36552a7c49f5c3de4508940e9de8c28f5824d90d42e4ddcec8","result":{"confidence_high":147.91,"confidence_low":143.61,"sample_count":null,"score":146.09,"score_display":"146.09","source_stated_rank":74},"run_fingerprint":"a4b45119fc464af264d74feb1a5275138f632e3882e9dab5915bf587cc040d7e","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_eebba79f988efd30ca30bc67","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":25,"at":"2026-09-09T23:36:58Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-09T23:36:58Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5f25e4b5ce726d6a56db49a5612c5874cb1b97adc096beaa87217d238b6dd281","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash (Non-reasoning) · intelligenceIndex"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash-0420-non-reasoning","upstream_model_label":"DeepSeek V4 Flash (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"b719fbf84294a83c93073d0131c3e712e3aa66a7c536614334f0ddaf98dc589a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.876906,"score_display":"18.9","source_stated_rank":null},"run_fingerprint":"153b46e221df27c33f397450f3710b56dbeebb1dc787a32ac922d6e7edc60a39","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_eda89840d177ea9f880a9299","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6157185449ef520abffc5c8b07b4b8498e0df01c90ac26fc827ff986bd446e66","metric_details":{"license":"MIT","variance":4.405695824738089},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1442.5375631012214,"confidence_low":1434.3097325032484,"sample_count":51648,"score":1438.4236478022349,"score_display":"1438","source_stated_rank":98},"run_fingerprint":"5fe0c6643b02f6517b58eb2cf954ce84be76a4e5565187f0c3a08d142db27242","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4c4c17865e3eb84674f26977c973e67d2e700b89e587e9f29b5189632c76d188","metric_details":{"license":"MIT","variance":4.26071249523967},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1440.0652166449593,"confidence_low":1431.9738999171873,"sample_count":51850,"score":1436.0195582810732,"score_display":"1436","source_stated_rank":103},"run_fingerprint":"341b9d0181296d279fac2b63810f3a0dc3f15b38c066580bc7890b2449d6d15b","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bddadad5c1a9d66ae999b217","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a865b984124bfa2e23c8a8eeb0695ede7ef6e768d78e4ad1f114167c5a45f926","metric_details":{"category_scores":{"Agentic Coding":37.626,"Coding":69.22800000000001,"Data Analysis":68.02300000000001,"IF":63.137499999999996,"Language":70.12400000000001,"Mathematics":79.64750000000001,"Reasoning":70.58175}},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.48110714285716,"score_display":"65.48","source_stated_rank":63},"run_fingerprint":"694c8ac2a61a5ed85d784e466a8ab7c9b384649b614720dfa578c1e2f7050000","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4205393e3ffc155c0364a879","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_1f4bfa5486eb4deec4942449","split_or_window":"provider-report-no-tools","unit":"percent","version":"HLE original"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":164,"at":"2026-04-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"stale"},"measurement_id":"7ea3e8bc80cb124094bac0118ba8cd4f238750b1cff0bcd9f531e6c85cf2cde2","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table · HLE (wo / w tools)"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"DeepSeek model-card HLE comparison; tool status reported; exact dataset revision and judge not restated","reasoning_effort":"reported","result_published_at":"2026-04-24","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9d79f00315c23bc47d2293c9003ea42de6ca857854c04113ff97996ca36b3878","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.8,"score_display":"34.8","source_stated_rank":null},"run_fingerprint":"4e2ebaaa6dae65fda59f90543a4b050e62959f217320f102bd15cc717bda481e","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table · HLE (wo / w tools)","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a5bf84569eb675351feaaf13","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"199dc018f577c26dceabfbf6f4922d8465e13455c8f3c6de2e1d9d6c96ab1a26","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":null,"num_parameters":290944616402,"pull_request":11,"result_file_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/resolve/refs%2Fpr%2F11/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash"}},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.8,"score_display":"34.8","source_stated_rank":33},"run_fingerprint":"34bfe8a3614c7667c31da5ffe3cda2822d0dba84111c57edda90c5d29b0c904c","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a6d9791c5af6765cb340d20f","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":164,"at":"2026-04-24","basis":"evaluation_at","evaluated_at":"2026-04-24","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5988560cf4e306bc0e20f90f3d12c2361e4641acec6e0f0d9ca5eec4adc83a4e","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/deepseek-v4-flash","source_model":"DeepSeek-V4-Flash (Think-Max)"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7c25b8318a826164a3c354f2810f2b67b5debf45128c2a34b217a7dcd9a5f4f8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.8,"score_display":"34.8","source_stated_rank":19},"run_fingerprint":"4ee82464bea6406a69c0d301079cd647b7af37f1471d284f9d2e54e35340ad4c","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_277305b76ab123eb94f5d2b8","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5dc5214f4902e17be96d77ff","split_or_window":"provider-report-with-tools","unit":"percent","version":"HLE original"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":164,"at":"2026-04-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"stale"},"measurement_id":"3f5cb5796101f8bd36cfc96b5d593a7c30220db265e646603a414320c724e3f3","metric_details":{"comparison_warning":"The model card reports HLE and tool status, but does not restate the task count, revision or judge; no unsupported 2,500-task sample size is inferred.","comparison_warning_code":"hle-model-card-protocol-incomplete","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table · HLE (wo / w tools)"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-model-card-protocol-incomplete","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"DeepSeek model-card HLE comparison; tool status reported; exact dataset revision and judge not restated","reasoning_effort":"reported","result_published_at":"2026-04-24","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"tools enabled (unspecified)"},"run_count":null,"scaffold":null,"tools_allowed":"tools enabled (unspecified)"},"protocol_fingerprint":"c1a05fd6d83a79a2bdd392afd643d80767c938453c8d68285a5d0cd4f529ee18","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.1,"score_display":"45.1","source_stated_rank":null},"run_fingerprint":"99d49338f7026c8a6ff41b595cd33a35e1b38b9e96a096a954d272faa5ea928c","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table · HLE (wo / w tools)","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a5bf84569eb675351feaaf13","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":25,"at":"2026-09-09T23:36:58Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-09T23:36:58Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0ca563381404f3bb87a0e833778aaa8ca28c1684769ad2cac689ec09714a8917","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash (Non-reasoning) · hle"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash-0420-non-reasoning","upstream_model_label":"DeepSeek V4 Flash (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"79335436b03dfb862b6194cf674507844905c3d45289028b9afdf4387cc3f45d","result":{"confidence_high":8.991959,"confidence_low":6.728046,"sample_count":2158,"score":7.784986,"score_display":"7.8","source_stated_rank":null},"run_fingerprint":"e06c4d8352f94395bb4a55ac011df146b2f51f389a831f127f434bf1ce7b17dd","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_eda89840d177ea9f880a9299","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":25,"at":"2026-09-09T23:36:58Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-09T23:36:58Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4fa238e16b4595bade2d8d75da1795ab83cbff2ae4844e730cd992272075e4a9","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash (Non-reasoning) · gpqa"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash-0420-non-reasoning","upstream_model_label":"DeepSeek V4 Flash (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ffe952529fe1690559e859136c0741080672196f94432543cdc2b55e58ada696","result":{"confidence_high":77.438359,"confidence_low":64.971135,"sample_count":198,"score":71.616162,"score_display":"71.6","source_stated_rank":null},"run_fingerprint":"c48da74884a27ae1895b897d5a74bdac3407a5a6f03cf389a5d41977e6eb3c00","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_eda89840d177ea9f880a9299","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6c378ca2a1fbac02190987c82e0a8af727aef61feffa8f05fd9d992147b33a1c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4-flash_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6c44e719da4d5dc2621f703a077ae100a9d0647af368fa7a84c8f704cd4f32df","result":{"confidence_high":15.578452,"confidence_low":3.106955,"sample_count":71,"score":7.14285714285714,"score_display":"7.1","source_stated_rank":104},"run_fingerprint":"f3e455ee75f88c4adefe24f0eba8e987889bd03a8c9a63e7c758498ddb9ce7ad","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_277305b76ab123eb94f5d2b8","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"55533073e4c1c1df05a4f439d7f5275cca8df8d37b2de7ae0d73c5c3452286a8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash (Reasoning, High Effort)","source_effort":null,"source_model_version":"deepseek-v4-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"62e6ed4246f449ac2117efb604eca4b27c0a18bd5617eca17690c042c76e1555","result":{"confidence_high":10.584558,"confidence_low":1.053583,"sample_count":71,"score":3.42857142857143,"score_display":"3.4","source_stated_rank":120},"run_fingerprint":"db6458c41445ca5c6f84e2188456ec62f457a0e5aa3d049cbd78db17c95a7ca3","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":25,"at":"2026-09-09T23:36:58Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-09T23:36:58Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b503b017642d06e105ee44de97d1a223338e869ffa3585971302c552aae0d465","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · DeepSeek V4 Flash (Non-reasoning) · critpt"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"deepseek-v4-flash-0420-non-reasoning","upstream_model_label":"DeepSeek V4 Flash (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"3ac97c3206650b74f25224218dab2ddb276c85687e3e1ab6c72733caf61af50f","result":{"confidence_high":5.730683,"confidence_low":0.013504,"sample_count":70,"score":0.285714,"score_display":"0.3","source_stated_rank":null},"run_fingerprint":"29e5d7206e0d2ed52ec89713a13e29db658fbaf682f4940faff5e927361dcd83","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · DeepSeek V4 Flash (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_eda89840d177ea9f880a9299","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f0812a8c38ddc282a9a7773da55ed322370e106c66f49fb86c98011524d4b39e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-flash_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"2221c745b377caaed9b0d2163c8c860d834304111b9d41acb2983d63b8dd702f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.3,"score_display":"46.3","source_stated_rank":61},"run_fingerprint":"05fe591c6e51b8d439ad8fe1adeaba7f1a40677f698ec8c5cbaa7a87e38f54fd","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1f7d663cf31b9ff28820da09","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_332cbf0e979daa9e173de9f4","split_or_window":"2026-05-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-29T23:53:02Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"d4102e0fdbef1329b958da4b9a2ad911e0afc0f96e4e194b1c76a4233a5720d2","metric_details":{"comparison_warning":"Official post-release SWE-rebench window; use only for same-window comparisons and never mix it into the canonical rolling-window rank.","comparison_warning_code":"swe-rebench-post-release-window","model_release_date":"2026-04-24","pass_at_5":47.61904761904761,"potential_contamination":false,"sem":2.4281045302822784},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-post-release-window","comparison_warning_code":"swe-rebench-post-release-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","supplements_range_key":"1772323200000:1778803200000","upstream_model_id":"DeepSeek-V4 Flash [high]__tools","window_from":"2026-05-01","window_status":"supplemental-post-release","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"3ddda07372b15613c728aba016bbc85e044ac6f484d9590196b5cd0f7c102c71","result":{"confidence_high":44.75908487935327,"confidence_low":35.24091512064673,"sample_count":null,"score":40.0,"score_display":"40.00","source_stated_rank":null},"run_fingerprint":"3844059c64b7ee0430d0b146b0378a0e4ae15763234634cbff5ad9b5e0962505","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-post-release-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"da59e3af7526b4209f0e0293fbb2c9302ddb22018548b82b7ccaa5dad06a63f9","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-04-24","pass_at_5":57.27272727272727,"potential_contamination":true,"sem":0.9706434774573924},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","potential_contamination"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"DeepSeek-V4 Flash [high]__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"3f3ba7fea411425c341e66a052563aee36c46bb459ea8d03c0445ec138bc3ef3","result":{"confidence_high":40.266097579452854,"confidence_low":36.46117514781988,"sample_count":null,"score":38.36363636363637,"score_display":"38.36","source_stated_rank":null},"run_fingerprint":"2a6c718340b3371876cbe46b5fb83b947057b6a09b581e179387be0ad8b3f648","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"pass_at_1","name":"DeepSWE","release_id":"release_4731f79a89e61b8ad678b622","split_or_window":"113-original-long-horizon-tasks","unit":"percent","version":"DeepSWE v1.1 live leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":36,"at":"2026-08-29T23:52:22Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:22Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-22T06:27:15.860279+00:00","status":"fresh"},"measurement_id":"9238619bc18e226d74a594c59bb2e9ffffbce91fe93441b3a9b4e2e2d62f6049","metric_details":{"ci_attempted":452,"ci_half_points":3.5669502150324286,"ci_method":"95% run-to-run: SE across repeated whole-benchmark passes (1.96 * std(runs)/sqrt(R))","ci_passed":241,"comparison_warning":"DeepSWE generated_at timestamps the live leaderboard snapshot, not an individual evaluation. It is used only as a freshness proxy.","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","leaderboard_date_is_proxy":true,"leaderboard_generated_at":"2026-09-22T06:27:15.860279+00:00","mean_agent_steps":152.87610619469027,"mean_cache_tokens":19719518.867256638,"mean_cost_usd":0.10023560370265487,"mean_duration_seconds":1438.9961937522125,"mean_input_tokens":19829931.451327432,"mean_output_tokens":107687.02212389381,"median_agent_steps":148.0,"median_cost_usd":0.09124898719999996,"median_duration_seconds":1383.7182735000001,"median_input_tokens":17312559.5,"median_output_tokens":104491.5,"median_output_tokens_to_pass":100483.0,"median_peak_context_tokens":201636.5,"n_attempted":452,"n_passed":241,"n_tasks_attempted":113,"n_tasks_passed_any":91,"pass_at_4_points":80.53097345132744,"scope":"Every DeepSWE rollout across imported Pier jobs, grouped by configuration (harness + model + reasoning effort)","unit_definition":"pass@1 is attempt pass rate over scored rollout attempts. pass@4 is tasks with at least one passing rollout divided by tasks attempted. Context-window failures and agent timeouts are scored failures; provider/verifier/network errors are excluded. Efficiency aggregates are over every scored attempt."},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"deepswe-owner-leaderboard-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"source_config":"mini_swe_agent_deepseek_v4_flash_max","source_model_version":"deepseek-v4-flash","source_reasoning_effort":"max","source_row_date":null,"task_coverage_status":"complete","task_set_size":113,"upstream_source":"deep-swe"},"run_count":4,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools in an isolated sandbox; no internet"},"protocol_fingerprint":"e858d0bfc6745368cc291d05a20f98f384295dfe782269142355d5d72adf3db8","result":{"confidence_high":56.88553428582889,"confidence_low":49.751633855764034,"sample_count":452,"score":53.31858407079646,"score_display":"53.3","source_stated_rank":46},"run_fingerprint":"af904b09b8ddd41cddb0dee730909017d6160fc6fbcbb23b39d6d457da301045","source":{"content_hash":"a7c15d66288fd249c020b9931c017b92d1a3b90e480b3ff34974b752bd030019","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-30T18:53:13Z","homepage_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","id":"deepswe","last_fetched_at":"2026-10-05T12:33:23Z","license":"DeepSWE repository Apache-2.0; aggregate leaderboard facts","locator":null,"name":"DeepSWE official live leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_277305b76ab123eb94f5d2b8","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5ee5141734ad53959a6ec003047aec0b6d40151a81831366405e90a01f87d578","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash (Reasoning, Max Effort)","source_effort":null,"source_model_version":"deepseek-v4-flash_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1b7162bf1583ff5201fcc2c31296c7dc34046178f2bc03b1dd3fd3a5f4c101d8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.907407407407405,"score_display":"44.9","source_stated_rank":124},"run_fingerprint":"f84ae7e85f4172c01a9f538446ddb3e1c752fdc5970ac1169ea7f1f5c0fd1c54","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_277305b76ab123eb94f5d2b8","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3bd1b5c0797163887973b40448770d0863097a4be3837ded44291eb89a690312","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"DeepSeek V4 Flash (Reasoning, High Effort)","source_effort":null,"source_model_version":"deepseek-v4-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"f6d93e7155ba51e2aef37374534339a9af09a7be0e333ded76c86e5492917333","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.0138888888889,"score_display":"42.0","source_stated_rank":139},"run_fingerprint":"2afc5d0dc2ba40ff8cfa6e1b6bca0494dd7ed2874e14b3873e5ce1b85b16203f","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"db4d28390da06b0159148a82c02f39fa125b948ef36eb4a166a9c97a633456a3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.005179","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"deepseek-v4-flash-high","source_effort":null,"source_model_version":"deepseek-v4-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"04e6ac1332540961c0b5a663bb13fe5851a752dec7ac4d9a7e4c507ff8094681","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":678.2,"score_display":"678.20","source_stated_rank":80},"run_fingerprint":"f4294a79de1cbd993c38bc737573af43e380f067ce8912214ca7f8f78d07a34a","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a19e50fef2bcb24ce2f0e26638bf8851483478f388eb3c99a08e071bbbf7cf00","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.000882","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":null},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"deepseek-v4-flash-no-thinking","source_effort":null,"source_model_version":"deepseek-v4-flash_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"8587da9ad6e4f07882ce202d5a8016c54ecb11858ff1f3b9cf899c20ca846c36","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":324.98,"score_display":"324.98","source_stated_rank":110},"run_fingerprint":"2d3b66fa638cea45cfc6680e7eef91a95075297968a5622294594217e9a5f5d3","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_eda89840d177ea9f880a9299","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"496d4a2b3a579e99a5dda3207122ee44c80eddde5505a85f7d91659580bbd5a5","metric_details":{"license":"MIT","variance":12.52062726699747},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"deepseek-v4-flash-high-preview","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bad904fb2382aea581f3f358c46a6f4150020dd47c297f0ebf32306d3c4664ad","result":{"confidence_high":1437.1923621865594,"confidence_low":1423.321893693328,"sample_count":10158,"score":1430.2571279399435,"score_display":"1430","source_stated_rank":78},"run_fingerprint":"acc6713dc5db93445253a0ace81820bfe22f687e43596254505f2cc2a9a21e03","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8ec0ff20e3b6ce062a9951bdbec047c113e1d0e4a6b47dad1da8526cf184779b","metric_details":{"license":"MIT","variance":22.230560395176134},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"deepseek-v4-flash-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"02bf5620ff818779cd339408d78641c983f4eb51e44c9a38e1e6eb314913a6f4","result":{"confidence_high":1590.2237819135546,"confidence_low":1571.7415978089612,"sample_count":5040,"score":1580.982689861258,"score_display":"1581","source_stated_rank":31},"run_fingerprint":"b22402179584255b92bbe2310c7ac2ce6f55d919f2d20c71b1a7f3d2d8ce2499","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ba8072d6201e6b2dce8235e62e471fdf6ba52916fe1fd866cddf690de9656011","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-flash_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"fdc3b48c5670835b2965c3a009407d78f85c18538183e30822468a533d245a75","result":{"confidence_high":1438.07,"confidence_low":1423.79,"sample_count":9493,"score":1430.93,"score_display":"1430.93","source_stated_rank":77},"run_fingerprint":"e95ba0975e8fec50e7f0332ccf9f875ebe7487fa3a841632bd6ad4aafd8a9051","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7269b8fba61408041f04fc75e86e439c5aa22c870e3392cdefe659903826f247","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-flash_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"c7df614619f26d191e0477590f0757699f3397570d859494fc0fdbb2c29afa98","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.63,"score_display":"45.6","source_stated_rank":88},"run_fingerprint":"fd0fbcc329ff94f82ec5928dc747fa173ed2ee49b205a814a43190898a6cc7ad","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_277305b76ab123eb94f5d2b8","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b5f02e2bf3eff14ff36b5c6fc44eb73b880a2b16ab659843663dfe3b7a237089","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-24","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"deepseek-v4-flash_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"326e96eb48d70643aad5b3829717a455387f8bc8af776cb8ed634dfb9f6dab83","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.76,"score_display":"43.8","source_stated_rank":93},"run_fingerprint":"4f01706cb020464ec66ca27afc2aff3182efe2e261f1fed6c807cb0b6d6e0c3e","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_ec6f9dd70348c797be90f74a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_83e2a53bddae6e68d7ed56ed","split_or_window":"provider-reported-terminal-bench-2.1","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":66,"at":"2026-07-31","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T18:49:20Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-08-13","status":"fresh"},"measurement_id":"ca4f42e7973a5ae25c4a66240134a0079e353cd3d069bba98423059c5085ce4d","metric_details":{"comparison_warning":"Different agent harness from the Terminus 2 run; both measurement series remain stored separately.","comparison_warning_code":"terminal-different-agent-harness","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Immutable README · benchmark table and note 1 · Terminal Bench 2.1"},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"terminal-different-agent-harness","judge":null,"run_config":{"curated_transcription":true,"evaluator":"DeepSeek","evidence_verified":true,"freshness_proxy":true,"protocol":"official comparison score; exact effort, sampling and harness not restated for this model","reasoning_effort":"reported","result_published_at":"2026-07-31","source_content_hash_method":"raw_bytes","source_content_sha256":"61755d88e95789fcd7a36f50892f97bba977a30fc99d0f2907ab787ed10b0e66","source_published_at":"2026-08-13","tools":"terminal agent toolset"},"run_count":null,"scaffold":"not restated in comparison row","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"2856f01ce5091a1b0fbf9afd2ac976cbff4761be067a817d79b63afd325fbd4b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.8,"score_display":"61.8","source_stated_rank":null},"run_fingerprint":"d8e97704de6d2539f0af3ce07dfa4fa815d05fdbabae8e24d0289116be968379","source":{"content_hash":"39b1a28d9b80da4fc9b60b5761dc77e3bfb8a523c51b24b53658814b279ecfeb","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-03T09:21:22Z","homepage_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813","id":"deepseek-v4-pro-card","last_fetched_at":"2026-10-05T12:33:37Z","license":"Model card; minimal factual score fields with attribution","locator":"Immutable README · benchmark table and note 1 · Terminal Bench 2.1","name":"DeepSeek · V4 Pro 0813 Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro-0813"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a5bf84569eb675351feaaf13","provider_config":{"source_id":"deepseek-v4-pro-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"C","evidence_rank":4,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"secondary-re-report","verified_status":"secondary-re-report"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"0c6f6dcfecc265f7b63ae4e140d08e2d49cb61563b28f532dd9d760fb8f7e648","metric_details":{"expected_trials_per_mode":261,"normalized_gain":23.7,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":17.2},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":53.2,"confidence_low":36.2,"sample_count":87,"score":44.7,"score_display":"44.7","source_stated_rank":16},"run_fingerprint":"9a1d3bb48584137ac53d1dd9f027f9f5bc3eaf08fbd57ba1e1ae72f4335950cf","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4ce18cb91b2b4b478d573def","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"21060623b340b8e0ef75ace5e2e80a94d488b2c3bea83280d2ea264d4dba4762","metric_details":{"expected_trials_per_mode":261,"normalized_gain":23.7,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":17.2},"model":{"id":"deepseek/deepseek-v4-flash","name":"DeepSeek V4 Flash (2026-04-22)","provider":"DeepSeek","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":34.9,"confidence_low":20.1,"sample_count":87,"score":27.5,"score_display":"27.5","source_stated_rank":16},"run_fingerprint":"4b47bec09e1c1c7372423161cfad518ac839bef1550b21f58637c5cdab54268d","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4ce18cb91b2b4b478d573def","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"527397e15641897899bff043258f36f81da28762ffe366808de8d70f3273fcb2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (Xhigh) · intelligenceIndex"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7","upstream_model_label":"Grok 4.7 (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"5370a1617a1de198c395c5886468c44d0b922238443450f5b331707c3080dbda","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.446551,"score_display":"46.4","source_stated_rank":null},"run_fingerprint":"2e9546e02c1351cb9e42a86cd288f625b6f7d5df23abdf808fad5dfbf4989fcb","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"18e9195e61b066d60a2be1b0a9f5c6c71457f2670367e5dcfd41ae08c99cec99","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (High) · intelligenceIndex"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7-high","upstream_model_label":"Grok 4.7 (High)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"19b80cb557ad81c301170295b104988667042ed4af50ba61379f1bafe7041854","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.332177,"score_display":"46.3","source_stated_rank":null},"run_fingerprint":"23fe300c736020a54144158c3e623fc8fdc064c8f0ebd722315bd236da391e45","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (High) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b1f23152c5d73c89b843de81","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-01T22:31:28Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:28Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e3c5a00ed1dc1a6a58a59c30f96780dfa239e241e61b0b187bd8d96a9dd2667d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (Low) · intelligenceIndex"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7-low","upstream_model_label":"Grok 4.7 (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"42a3c73458f6cd077e1040032f0d3320456f522b53ee2f0ee21003cdf63aa5dc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.216656,"score_display":"42.2","source_stated_rank":null},"run_fingerprint":"6d198cd521385090956262b15093926e8c8d567f644c6ff6df947df78a59f964","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_cc4e54642b60580b7257387a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5ae7b071a8175adab5f8f1e2cdaa2c44ed9ce94439ced084c8d2edbe9a4f6d42","metric_details":{"license":"Proprietary","variance":15.708410662535465},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1450.2252810222135,"confidence_low":1434.689102251527,"sample_count":6405,"score":1442.4571916368702,"score_display":"1442","source_stated_rank":91},"run_fingerprint":"e9d34b52c7c13cf3a9c5fcc3586e8d155fc4edf0172a913359b18d5475784bcb","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"6c67061ad5a018a3a40db2b529fc2f698080727f4823f495f02097e72fa87584","metric_details":{"category_scores":{"Agentic Coding":53.989666666666665,"Coding":77.1585,"Data Analysis":76.88333333333334,"IF":75.27525,"Language":80.13866666666667,"Mathematics":95.678,"Reasoning":82.65375}},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.39673809523809,"score_display":"77.40","source_stated_rank":25},"run_fingerprint":"8b6e1f4e870a5da7545ee6e348dd443362572c3509f12ab255e9bdf10c9eaf12","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7655be84c9c7dc4a81883691bcf614b4c463d385578603fa2b5ed81065569f32","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (Xhigh) · hle"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7","upstream_model_label":"Grok 4.7 (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9e8f1c5989677f15ba51b94e8795d0600a32cf994a948dfa1aeb5baf5eca1423","result":{"confidence_high":45.241825,"confidence_low":41.066145,"sample_count":2158,"score":43.141798,"score_display":"43.1","source_stated_rank":null},"run_fingerprint":"3c6d938bb15f317fef62d3c9b285c11f0c148f8819042ffe8037e567251784d8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c3352dc8a350f9bb84ff34621a0414407502762e08932e01cd6e2e63e6abd3e3","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (High) · hle"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7-high","upstream_model_label":"Grok 4.7 (High)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"59458c515ba80a385e08bb82a882400be7d9385e0adde61a743eb4f679874bbc","result":{"confidence_high":44.404039,"confidence_low":40.238683,"sample_count":2158,"score":42.307692,"score_display":"42.3","source_stated_rank":null},"run_fingerprint":"2105ade7acb22a444b1178f22dd3b3380f10359263be003048ff5160e2253ec5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (High) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b1f23152c5d73c89b843de81","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-01T22:31:28Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:28Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2014a07af187585c8ba7c98d5a844fd578fbfef8adb970f4abe8052a5a5520a8","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (Low) · hle"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7-low","upstream_model_label":"Grok 4.7 (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"755e881ac900236727e5c9e6026e840c04ca7d12902e43102ed58a7fe349f486","result":{"confidence_high":41.23355,"confidence_low":37.118241,"sample_count":2158,"score":39.156627,"score_display":"39.2","source_stated_rank":null},"run_fingerprint":"f92aac059709eeea992ea7b0ea0e856991a5f2a2ed57f44d1c25e2194c25e9df","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_cc4e54642b60580b7257387a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T21:49:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T21:49:22.000Z","first_observed_at":"2026-10-01T20:29:30Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5f7b54d8ae034c0a32a0180d8074222e37f36bf07ae7950f76a5c3dfe86bdb4d","metric_details":{"best_score_across_scorers":"0.56","model_release_date":"2026-09-21","stderr":1.5704987954361718},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"6izDCw5ygCLZJEKPS8jV5T","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"43c130d8c464e692585f692717ab455f3f31e03573c611e2a24208f0bded20af","result":{"confidence_high":59.07817763905491,"confidence_low":52.92182236094511,"sample_count":1000,"score":56.00000000000001,"score_display":"56.0","source_stated_rank":18},"run_fingerprint":"ddef77ddb5ea177b9f3695b0d6f5f9e124e32b68e0c0d8ac1dd4a57ec78c7763","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":12,"at":"2026-09-22T21:49:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T21:49:22.000Z","first_observed_at":"2026-10-01T20:29:29Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2fa8a7c4016d9ea00346ee1242ffe9a859ffe2ca0de6fb83cdb272269a9c529a","metric_details":{"best_score_across_scorers":"0.92739898989899","model_release_date":"2026-09-21","stderr":1.5505824295534454},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hxAhYJfXdPNanxatXpG5hC","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"c7166913d39cf25ff6883b56b97c6c4e2c2069c08eb39b06111cc2a769676565","result":{"confidence_high":95.77904055182374,"confidence_low":89.70075742797424,"sample_count":null,"score":92.73989898989899,"score_display":"92.7","source_stated_rank":23},"run_fingerprint":"2d6b14008c5f08d87d86d29d637c2c56249fbed180e117b2f345a4ad7f70fc29","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T21:49:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T21:49:22.000Z","first_observed_at":"2026-10-01T20:29:17Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"83ed3245d2ad7f84bbff25fec829537ea97fe9dfa94e202e5fca422d398c3157","metric_details":{"best_score_across_scorers":"0.9805555555555555","model_release_date":"2026-09-21","stderr":1.4299486306926388},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Jhgwm7QpspGcvm7DUVqUDg","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3b87eef345d7e339a4cf86dd308fb014be73564b5beafbb401d4930cd7ab9441","result":{"confidence_high":100.0,"confidence_low":95.25285623939799,"sample_count":45,"score":98.05555555555556,"score_display":"98.1","source_stated_rank":23},"run_fingerprint":"189b69ac573644379bb46bdb05b265c276884c87746388badbe4b11d364acb26","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T21:49:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T21:49:22.000Z","first_observed_at":"2026-10-02T16:27:53Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0e0eac38308d63f4db28f4fccff574115da86bae2577fb4215242593b9700768","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.5298245614035088","leaderboard_private_problem_count":285,"model_release_date":"2026-09-21","public_example_count":10,"stderr":2.9616712041900364},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"F4unBbW5M4VJogzCq4vGAb","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Grok 4.7 (xhigh)","thinking":null,"upstream_model_id":"grok-4.7_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"ad64bf406a59155c8c66d3acacd28cb226ec54a10ee69ae99d3b1a4369969769","result":{"confidence_high":58.78733170056335,"confidence_low":47.177580580138404,"sample_count":285,"score":52.98245614035088,"score_display":"53.0","source_stated_rank":46},"run_fingerprint":"d2df4effcebe5204454017dab41d9f025ecba9b4c5cde00ea8c19e614bdc6723","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T21:49:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T21:49:22.000Z","first_observed_at":"2026-10-02T16:27:53Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"489360a09319b3c09ccbbca767f2acea670d94de3a46e603867668547b7a2d0e","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.17073170731707318","leaderboard_private_problem_count":41,"model_release_date":"2026-09-21","public_example_count":2,"stderr":5.949419959829496},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"Ymgj5VeEceBbzsLq6xEZRi","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Grok 4.7 (xhigh)","thinking":null,"upstream_model_id":"grok-4.7_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"8f425196ee2bffb2b6982024f14ea775092e191848a81f4c192683d2b8aeee52","result":{"confidence_high":28.734033852973127,"confidence_low":5.412307610441507,"sample_count":41,"score":17.073170731707318,"score_display":"17.1","source_stated_rank":48},"run_fingerprint":"de0fc9c987ee1e5970534efd41f87fbc95306e14029c532d3401610cda6efa83","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-01T22:31:28Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:28Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7f78af11b0e549abe8e368aec0a5298aebd5d7fe7ff707a0661395e4210b3dc0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (Low) · mmmuPro"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7-low","upstream_model_label":"Grok 4.7 (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"401dad5b43f28870fdaedfb934dbab6b034cf1297a3bb18f3cd663d53d96d4d8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.150289,"score_display":"78.2","source_stated_rank":null},"run_fingerprint":"db4fb94e2da2ff8ea7344822b2b7d5d5ecdf5bb4b41beb27d06f1f84ee598751","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_cc4e54642b60580b7257387a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"657b6ebc99414f0c2da5714e0b224d4c70ad21f90714d5e9600739f96cdb11ac","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-21","notes":null,"upstream_source_url":null},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.7 (xhigh)","source_effort":null,"source_model_version":"grok-4.7_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"57af23c2e8298d8e13e192f7b62e4c014c47ab72ad176f3cde867d73c090844f","result":{"confidence_high":28.178675,"confidence_low":10.564331,"sample_count":71,"score":17.714285714285698,"score_display":"17.7","source_stated_rank":61},"run_fingerprint":"8da554d27f384e6314c4c5071a28c3380fe854718dc60b70b70b0d5a2120a796","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7524640ed3cf26d2d145215a743759699d4d04a9eae9b15c90627b25b0f5ae28","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (Xhigh) · critpt"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7","upstream_model_label":"Grok 4.7 (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"e7f1a25ef21e1d641d967fc880b01b99700fc1707fa84fdbc7ab885638fab288","result":{"confidence_high":28.262679,"confidence_low":10.525214,"sample_count":70,"score":17.714286,"score_display":"17.7","source_stated_rank":null},"run_fingerprint":"4c84da1edc879e0b2ee3b818bdae0e3e5e6cd5651eda2c8a0e8ab540df6abd86","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"904928f3d4680bc2845af3b2400a26237a38c7049a2ec008110492f319a84062","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (High) · critpt"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7-high","upstream_model_label":"Grok 4.7 (High)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7d9025be976b0e780a1e65b2335076d26e2cb61be7f69cdbb683e079ab5502ea","result":{"confidence_high":28.584452,"confidence_low":10.74514,"sample_count":70,"score":18.0,"score_display":"18.0","source_stated_rank":null},"run_fingerprint":"c5f6cfcca1fddb8050abdc6fbcc64112202ad6b15aa9ebdefd7b920dc1767071","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (High) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b1f23152c5d73c89b843de81","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-01T22:31:28Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:28Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"34ee176963989616952c8029cb92e58e3c911676b3269a44849bf20603500786","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (Low) · critpt"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7-low","upstream_model_label":"Grok 4.7 (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"26438e8d7e1521481a14c8f535e8c4e98b4a64203e7d3427dada41a33aa92526","result":{"confidence_high":23.337473,"confidence_low":7.324917,"sample_count":70,"score":13.428571,"score_display":"13.4","source_stated_rank":null},"run_fingerprint":"3df77c2fa5f527a3abd1e7af7732f6b39b9c17ad41547c41b96de7c32541a27d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_cc4e54642b60580b7257387a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T21:49:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T21:49:22.000Z","first_observed_at":"2026-10-01T20:29:23Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ba919f88b163c58b4f09ea5489b484726dbd64ec6f17800da04a28d0529eb791","metric_details":{"best_score_across_scorers":"0.38","model_release_date":"2026-09-21","stderr":4.878317312145635},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"MpkdmmouJFUbKycL72HmSE","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"dccf2aba2be0daeffc37607d4ee4af308b26ac91f777c93a161f14de7cc255a1","result":{"confidence_high":47.561501931805445,"confidence_low":28.438498068194555,"sample_count":100,"score":38.0,"score_display":"38.0","source_stated_rank":32},"run_fingerprint":"1a7f5c89662a6aa0915027afdd50a1a5e948a8fa26145b13fce6a4e286a5b27c","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"12f3a0c54e2bab73b7bc1b6000ffae82a5168a5ca390c6e74ece262746c78e13","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"6.01","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-21","notes":null,"steps_per_task":"88","tokens_per_task":"70141","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.7 Extra High","source_effort":"Extra High","source_model_version":"grok-4.7_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"d76edb669136191ca56483e710009f2af05879097c609c7d976025e3a2107356","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.3,"score_display":"46.3","source_stated_rank":13},"run_fingerprint":"683494021adccd90f6b450b88c49cf0af5a910c7a6f48295e16f8dc678525f7f","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7d1fcb5972b1853000a48e81d88156e83b7dad350523b043f43e48da407b44a5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"4.69","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-21","notes":null,"steps_per_task":"71","tokens_per_task":"56382","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.7 High","source_effort":"High","source_model_version":"grok-4.7_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"d08cc55e27cde7bbd44edf4b94cf6f58ccbd892082184e919549ec959718fddf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.9,"score_display":"43.9","source_stated_rank":17},"run_fingerprint":"d6b89ab5bffc5cea82ca5e65d1aad9f08ac19609b09f2032553c95ef4bbf136f","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b1f23152c5d73c89b843de81","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"038a595ac96f49be00df2383d4c02e1fac2b988306a3d2164e0d15a58419dfd4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"3.49","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-21","notes":null,"steps_per_task":"60","tokens_per_task":"36683","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.7 Medium","source_effort":"Medium","source_model_version":"grok-4.7_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"ae77b2de97c979a787676bffa91c1331c8cee55c2502fb279dfacd84efdc0ea5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.6,"score_display":"41.6","source_stated_rank":21},"run_fingerprint":"91d755f5a70c04a176dff2841ba91071c174d0f701371ae6d6487c23150d1400","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_e1c36aad3caca85bb4ebf35c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0e4a8893c1dde1acdfc3ead506319a3c31485fb541d81b20b84afda61e9c8708","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"1.58","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-21","notes":null,"steps_per_task":"40","tokens_per_task":"15677","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.7 Low","source_effort":"Low","source_model_version":"grok-4.7_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"0ccba1a6cb5314289df0a847cb4b7235c561efaa3f443ee84bdb8047bb95d2f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.1,"score_display":"33.1","source_stated_rank":40},"run_fingerprint":"6a800c79e9a90f477d116fad9dd79b4bf07728264a76f6e8abe1bebf8bb72540","source":{"content_hash":"287acd0bf312a7f77ff568b8e02288cc9523623fbc3eac0a5adf43a9e29a1bea","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_cc4e54642b60580b7257387a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":14,"at":"2026-09-21","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"a7df71be0f5e355f30742c1e7bd2b8039505758747079a281c8e76062a85b4e7","metric_details":{"benchmark_page_updated":"2026-10-01","cost_per_test_usd":12.707908,"date_is_proxy":false,"latency_seconds":2808.481,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/grok_grok-4.7","source_locator":"Results · Overall accuracy","stderr":1.987},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":null,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"873eea6b95c98deddafed96e09c6b266241e102d4b2e9078014b7ab4e765ef6f","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.7"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"fc11f94745a484923818346b8125fd3e786fe0b29e8b820543f9b68defd31975","result":{"confidence_high":61.61652,"confidence_low":53.82748,"sample_count":null,"score":57.722,"score_display":"57.7","source_stated_rank":16},"run_fingerprint":"4c915965446b0d37878538af3f12207dae86ab51208c2ed2bd4fa22387105ebd","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"bf906f954ce434c441a81712cded07c60608b26f9e483a241f8a9c1f3a40ff13","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-21","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.7 (xhigh)","source_effort":null,"source_model_version":"grok-4.7_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"47389a6dafe0e23d23289bfcd7f7146f7d04cc1cce5593fd22dcaa3810680150","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.4074074074074,"score_display":"57.4","source_stated_rank":21},"run_fingerprint":"811742c34d51d2ba201374fefbf8021cd9057ad53d5082b6a7060779ebaf441d","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fbcad69f0f0dad9a01799b4511ec93a32eb9c3c9408e52f5caae32e634609fb2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (Xhigh) · scicode"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7","upstream_model_label":"Grok 4.7 (Xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fb412ec74e4d5fa3644b92008c5d26e459b62f90432149d9acfeb10591114833","result":{"confidence_high":62.984009,"confidence_low":51.635793,"sample_count":288,"score":57.407407,"score_display":"57.4","source_stated_rank":null},"run_fingerprint":"0e8ad8f1631743d93a00c532687cd418bab1792202991b68290c07abe022ae65","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4e71f2a6a1145a150ac7e846328da1ff72a5e73e98f164f9a3c3132dca02eb7c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (High) · scicode"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7-high","upstream_model_label":"Grok 4.7 (High)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"947e6fbdbb0746d4d373a79b490e82a751e1d85fb8a3f35a2ab33687d75ba282","result":{"confidence_high":63.320632,"confidence_low":51.984475,"sample_count":288,"score":57.75463,"score_display":"57.8","source_stated_rank":null},"run_fingerprint":"9f2a1846590bd05214032d1cc6a22ca98e83476bc57cccdcbc75bb1baf05d3bc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (High) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b1f23152c5d73c89b843de81","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-01T22:31:28Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:28Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4487563dad3b13ed1af2443bf07ef6fd2536a6b2bc43ea5da8e96175055cdff0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · Grok 4.7 (Low) · scicode"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"grok-4-7-low","upstream_model_label":"Grok 4.7 (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"048fd8d42735716f4705d3da5dc8ea2c459ddd364af7a4ca07e9453c3c99a541","result":{"confidence_high":60.506878,"confidence_low":49.087368,"sample_count":288,"score":54.861111,"score_display":"54.9","source_stated_rank":null},"run_fingerprint":"b24fd60db5b66c38c2c4a3744316cdf93e654a52701482917d93cf5e33d1a3a0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_cc4e54642b60580b7257387a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a5e73222fa4ca952d6b32bff101d286d7277070d74621a8b99e2aef0f81e6291","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":15.82706,"date_is_proxy":true,"latency_seconds":2147.065,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.181},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":null,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.7"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"2a4839c158cacf2f3864707cf228b8dd22a82deda96b18746fcf002f732c0fdf","result":{"confidence_high":90.44976,"confidence_low":81.90024,"sample_count":null,"score":86.175,"score_display":"86.2","source_stated_rank":10},"run_fingerprint":"6aebf0228eff687f5a0df5dcf85528ddf705cdfe69e880f2b1a47d8a9a600e1e","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"12e82defdf80812c303801babdaf3895428b0f1fc8852a85fdf78f8dda2aaf9c","metric_details":{"license":"Proprietary","variance":36.046835298977406},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"grok-4.7-xhigh","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cdd50c3e62fe748a8b53e5ee7f3336b878e183b096f12c2629b8885b9bdcc853","result":{"confidence_high":1649.2918735621008,"confidence_low":1625.7570114705256,"sample_count":3216,"score":1637.5244425163132,"score_display":"1638","source_stated_rank":15},"run_fingerprint":"841067bcd8c61179cea2de506d317d81f8c511b7f406cdf3f506f706b36e7dc7","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fe8919b8575a71cd323dd2a558086044e9777c56e93a5cc553b50e34d0207505","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.076601,"date_is_proxy":true,"latency_seconds":797.952,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.498},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":null,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":0.95,"upstream_model_id":"grok/grok-4.7"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"73044f0a91ab9319dd4b653f6420304d316e2a98e09eeb7e35bc00da22d2fa87","result":{"confidence_high":76.34408,"confidence_low":70.47192,"sample_count":null,"score":73.408,"score_display":"73.4","source_stated_rank":22},"run_fingerprint":"615bd2da766a82341b3e9e92a7ade78c24cedc20a64afb584ab14f687ce21bc7","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e1a449d19ef4e9241d63cd75a6fc45ca8667f5376ca780713745558ea179e64c","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":11591},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Grok 4.7 (xHigh)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3316780fbe1486b7633a353eda80966cb27b6d28befcb8f84f41a67d5b620e5a","result":{"confidence_high":0.05640365107581135,"confidence_low":0.024323979592707053,"sample_count":1656641,"score":0.0403638153342592,"score_display":"0.0404","source_stated_rank":16},"run_fingerprint":"f2fbc5f3b115fd4a62dcd1f75536cf430cc0218cd4613e5a71a77db4c4536851","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fab441b438e8888e2bafbc16c130a05664b9fc9488ebc44754625a84b6448f7b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","reported_confidence_interval":"-20 / +20","source_locator":"Leaderboard models table · Grok 4.7 (Xhigh) · gdpval"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-7","upstream_model_label":"Grok 4.7 (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"25db147c4d18bfcbec2df927f55118eea22b865a7f02d304c0e2af9ee5d082cf","result":{"confidence_high":1715.03,"confidence_low":1675.39,"sample_count":220,"score":1695.21,"score_display":"1695","source_stated_rank":null},"run_fingerprint":"0b0c3772a99ff184baf0dc85f75341727e78396d1e8f4edc2daadf94135c9b49","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_1105264d4579e04e2c6a4c9c","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a1e7526463e3094f33fd5f397af1355ea8566063711e90f6307b5a8cda1d47d8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","reported_confidence_interval":"-20 / +20","source_locator":"Leaderboard models table · Grok 4.7 (High) · gdpval"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-7-high","upstream_model_label":"Grok 4.7 (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"3ca2a2975bc3568f0e10ac3f68085504550383053f0338ebb093b8878550ebc8","result":{"confidence_high":1730.18,"confidence_low":1689.39,"sample_count":220,"score":1709.78,"score_display":"1710","source_stated_rank":null},"run_fingerprint":"29177eb88744d074ad044d99d2eecc7b3154164e724d2fbda12e9ec8788cb03b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_b1f23152c5d73c89b843de81","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-01T22:31:28Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:28Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ecff52499dadbbdba54c0ff57e97be5c3301a572920a5982b2c4aa25b71d47d3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":500000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-21","reported_confidence_interval":"-20 / +20","source_locator":"Leaderboard models table · Grok 4.7 (Low) · gdpval"},"model":{"id":"xai/grok-4-7","name":"Grok 4.7","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"grok-4-7-low","upstream_model_label":"Grok 4.7 (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"ccf782bf974657d7997072d217d5fef8abdc542bff28b29053804c3700e2dc3c","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1573.62,"score_display":"1574","source_stated_rank":null},"run_fingerprint":"bf464d02e99ed894e113c7944fca3e86c9471d00aa86d31756b0586bb9864554","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Grok 4.7 (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_cc4e54642b60580b7257387a","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"42ee0a0b3903016d9158dcb1bc249c4f6e29a6e4bbf009093ffe0d7f5af9f405","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-06-01"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: MiniMax-M3; model release: 2026-06-01","source_accessibility":"Open weights (restricted use)","source_model_name":"MiniMax-M3","source_model_release_date":"2026-06-01","source_model_versions":["MiniMax-M3","MiniMax-M3_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b441953929d4de4def626d6d047a60c98a2773fde435aa08d7406eb967547ed9","result":{"confidence_high":149.82,"confidence_low":142.71,"sample_count":null,"score":147.03,"score_display":"147.03","source_stated_rank":64},"run_fingerprint":"51e585a529fd434bca9a100bfaa5bfbe37ebc968945e6a60d998cf11bca2caee","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a112784282b9946d07ab8413","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9b2d6b0f01a11b3f53b19523856fba27654ad334a39f021f321b578fc524c2f6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · MiniMax-M3 · intelligenceIndex"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"minimax-m3","upstream_model_label":"MiniMax-M3"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"a829f0596cd3a65ca3093096b4c3f3b329588cb775076cf503ef5af1ea27fec9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.220293,"score_display":"29.2","source_stated_rank":null},"run_fingerprint":"9fab07e91b08985a347e1c5d279fa9e3376631f265729a41372be3f8314303e2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiniMax-M3 · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18bf23dd51a56ac5e133a7c1","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3de0f3d65ecf3c61732c2b0638612d1327a77e1fb1370c2c2e3679c45d810af5","metric_details":{"license":"MiniMax Community License","variance":4.049855762356034},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1443.9997052880005,"confidence_low":1436.1111428991135,"sample_count":58757,"score":1440.0554240935571,"score_display":"1440","source_stated_rank":96},"run_fingerprint":"8e065231586029e1281d69d7c5ddda0611e4edeae307890bc1c5b98830c99677","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ab1b92201926a7b0c5c81541","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"92161db5a2b641c91af071e2375ed7fd4ade4db75c8d965fedbb3bf3fa9ec9a4","metric_details":{"category_scores":{"Agentic Coding":40.65633333333333,"Coding":68.2025,"Data Analysis":76.16633333333334,"IF":57.50825,"Language":76.83566666666667,"Mathematics":76.94749999999999,"Reasoning":74.48075}},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.25676190476189,"score_display":"67.26","source_stated_rank":61},"run_fingerprint":"0f25208f0d7285e020c80cf18852a27f0c0d81031304b96c1fb5a18122abb40d","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3197178f18af61acd90327e5","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"923365ef75766ba72b9ae2c512d937b173ec265a3369113d88d6fecad91145d3","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · MiniMax-M3 · hle"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"minimax-m3","upstream_model_label":"MiniMax-M3"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"95fd55967098dda9c6f1946dd619b87d734a567cdd5dc90b37fbb5dc50ee4299","result":{"confidence_high":41.046774,"confidence_low":36.934962,"sample_count":2158,"score":38.97127,"score_display":"39.0","source_stated_rank":null},"run_fingerprint":"417bd1a7ba3db686977fb741fab0f5f39f76795241c04cba8d0e040aad9ed704","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiniMax-M3 · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18bf23dd51a56ac5e133a7c1","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"88e8914ad6fe07cec89092158d15f86486dcee88d2a063f1b053eca290bd5529","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.0255,"date_is_proxy":true,"latency_seconds":278.908,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.443},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":92.424,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":512000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":0.95,"upstream_model_id":"minimax/MiniMax-M3","zero_shot_accuracy":92.929},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"20f98c60abd39100026d84d7c729b704bb5ce315c226ad23060d1c8ff15f4b93","result":{"confidence_high":95.50428000000001,"confidence_low":89.84772,"sample_count":396,"score":92.676,"score_display":"92.68","source_stated_rank":14},"run_fingerprint":"db12800bc1db0bd47194c057a657833ed1b288e73f3e0dd4790085349e6a6747","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6a5ac3a55dc02af104cdb177","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eab37a83baa487fe084c0b7097ad92fd3f5a1bba984b87ff1e5541826b7c064f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · MiniMax-M3 · gpqa"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"minimax-m3","upstream_model_label":"MiniMax-M3"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"86a2c6a011825e740edda36881916927c88252b7ce56ea2eeaeedc03cedf93ce","result":{"confidence_high":95.741769,"confidence_low":88.482692,"sample_count":198,"score":92.929293,"score_display":"92.9","source_stated_rank":null},"run_fingerprint":"cbb7f4d3912384df05333dab42eabf3f9ee587b6847921446eae7a1eb92727a4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiniMax-M3 · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18bf23dd51a56ac5e133a7c1","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":55,"at":"2026-08-10T17:46:29.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:46:29.000Z","first_observed_at":"2026-08-28T00:52:16Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9157ba814a5da89e5e3d573176829ed9f48cb276a3028f87a5ccae7b9cb4af5f","metric_details":{"best_score_across_scorers":"0.8131313131313131","model_release_date":"2026-06-01","stderr":2.7772533334218945},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"KM4G49tQZB8ce67RxYGyMy","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKM4G49tQZB8ce67RxYGyMy.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/KM4G49tQZB8ce67RxYGyMy.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"57a7eeb3f4e36aa530e0ca3126ef1c807de708900dd2fa2b9575c732eb3c4857","result":{"confidence_high":86.75654784663824,"confidence_low":75.8697147796244,"sample_count":null,"score":81.31313131313132,"score_display":"81.3","source_stated_rank":122},"run_fingerprint":"a30784d81471decee736312aab3c164f26b96193ce743071be408def14515aae","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e15386937757ec3e2088b88f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":55,"at":"2026-08-10T17:46:03.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:46:03.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b2fb942392a6203883b4c09f79f05f373a1f6e0653e8622bfd7d205eb4a83210","metric_details":{"best_score_across_scorers":"0.9090909090909091","model_release_date":"2026-06-01","stderr":2.0482086775424224},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hDESmyGFrbZDomjff3rqzD","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhDESmyGFrbZDomjff3rqzD.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/hDESmyGFrbZDomjff3rqzD.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"d1bb8d32d440405f3d93279e8b2babc0137fd475a6d6d437b26538ca7a952503","result":{"confidence_high":94.92357991707405,"confidence_low":86.89460190110776,"sample_count":null,"score":90.9090909090909,"score_display":"90.9","source_stated_rank":34},"run_fingerprint":"a63609b158b4fa7b5fe0501bdc36bfa8afd6c001655685a070f4df391539f2fb","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_21df3ae2aab654591d12e5ea","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5786925767b9533adeffd946b77d270eb9840bf8fa0437e6f671059f474659e5","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.004811,"date_is_proxy":true,"latency_seconds":41.39,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.364},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":512000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":0.95,"upstream_model_id":"minimax/MiniMax-M3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"caf7bf3725cc444797164e14bfb5d46672bde23e193a0a7d74913f6505e39454","result":{"confidence_high":84.93344,"confidence_low":83.50656,"sample_count":null,"score":84.22,"score_display":"84.2","source_stated_rank":66},"run_fingerprint":"9e67616f58b5c5c1176ceb19ddf776f1beeafd8823aa066e86ad009ea87b9ea2","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b30a7dc7d7d151c591b300b0","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:46:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:46:50.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b8db686af3bc86351d72cf91b449587556beb7cdc9196e4b0d067bff782e24f6","metric_details":{"best_score_across_scorers":"0.26666666666666666","model_release_date":"2026-06-01","stderr":6.666666666666668},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"97t77hRrNae6Va8fBqBhPy","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F97t77hRrNae6Va8fBqBhPy.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/97t77hRrNae6Va8fBqBhPy.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"217614212a82bf56f939040ce2555be0306ce9ff38db111469c28fc88105ccf7","result":{"confidence_high":39.733333333333334,"confidence_low":13.6,"sample_count":45,"score":26.666666666666668,"score_display":"26.7","source_stated_rank":225},"run_fingerprint":"0fbf122fb1d16710dda0051f1e82c336cceadbfb716e87c9e7f82ccfde622173","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e15386937757ec3e2088b88f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:46:09.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:46:09.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7a81da5ee741580a4edd6610e0e48f1c805ff96df97c63a7f05f951da3cac471","metric_details":{"best_score_across_scorers":"0.7111111111111111","model_release_date":"2026-06-01","stderr":6.83294324254051},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"79ZP6iwoBaF5t9a5VRqPnw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F79ZP6iwoBaF5t9a5VRqPnw.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/79ZP6iwoBaF5t9a5VRqPnw.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c3c419a472fb5660858b6e32a2c4528e8de862aadcdcf652d5679e1628864aaa","result":{"confidence_high":84.50367986649051,"confidence_low":57.718542355731714,"sample_count":45,"score":71.11111111111111,"score_display":"71.1","source_stated_rank":132},"run_fingerprint":"0b5b7e4db087e1f558b566bced2cfdbf2cd733109e488fd8daa176e800efb97f","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6ddbdb5bfac3554a32c4e9b7","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ee13bbe8f1ad0523c3d128ffac229d2b575a256db824c570899a1a63c91fb832","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.009431,"date_is_proxy":true,"latency_seconds":68.354,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.94},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":512000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":0.95,"upstream_model_id":"minimax/MiniMax-M3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"87c955f5c139fb1f98e39aece574d97d1b9d6b0f7f53f2a53f77c691c0f9a460","result":{"confidence_high":82.9984,"confidence_low":79.31360000000001,"sample_count":null,"score":81.156,"score_display":"81.2","source_stated_rank":41},"run_fingerprint":"149076533e4be4357a3d208e67cc9b349d867cfaf3db2a14b6644fccc03932cf","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d535e0c60ed655f9066eb7da","provider_config":{"source_id":"vals-mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a1394817e9149f086e3bc8cecbbb0034b587fb3d9990cf2ac5d2242eae39036b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · MiniMax-M3 · mmmuPro"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"minimax-m3","upstream_model_label":"MiniMax-M3"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"1af3051b1b83e71c5137e80dca0242feb0ab52c93e73cc85f4658980e075a6da","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.554913,"score_display":"78.6","source_stated_rank":null},"run_fingerprint":"5246d73899ea1c151d4ca4b49401c5760b246e459cbfec1a02abe9a59af94697","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiniMax-M3 · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18bf23dd51a56ac5e133a7c1","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ea7feed6ff66f7e79c39ac80a6b0caec4465f070baf27be67622e6639a084a55","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-01","notes":null,"upstream_source_url":null},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"MiniMax-M3","source_effort":null,"source_model_version":"MiniMax-M3","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"08f66ed0368be313673d385df06b549e24683c838971fb432897e2c1b129ff19","result":{"confidence_high":10.98928,"confidence_low":1.190959,"sample_count":71,"score":3.7142857142857104,"score_display":"3.7","source_stated_rank":117},"run_fingerprint":"6ac71cb07f29e5a616803fe772ff4b52d4aa659f339ec572293c622a408a31d3","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f3c7a95c208191d57e63dc2e","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"48a343aa95bf08e23a092c38d3bd49f9c61e17f8feeca3f624d35bc3833618bc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · MiniMax-M3 · critpt"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"minimax-m3","upstream_model_label":"MiniMax-M3"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"bae4d29c5c34b9998e887c89654a1952a17a88d6034ff7f68eb65b549213e600","result":{"confidence_high":11.062365,"confidence_low":1.182224,"sample_count":70,"score":3.714286,"score_display":"3.7","source_stated_rank":null},"run_fingerprint":"8eb1322acbcbc85010defb0525ce1d4feb5815ac7ee793c2e510ed4ddd8bedb0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiniMax-M3 · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18bf23dd51a56ac5e133a7c1","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"40bf9d6a0afa7a214790cf1c63305decfa85b214580088af675ef2ecdb7beb45","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-01","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"MiniMax-M3","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"675ec8382f52f65e03381904418d26e916a0f19bc9b8754d277d20c026700fc7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.8,"score_display":"45.8","source_stated_rank":63},"run_fingerprint":"dc7a1308827a3e9c23c8bad78f1e3af0640d34369660385d19b5e4c5ec6a8f55","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8621ac6c3391417ec3519f00","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:46:41.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:46:41.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3fcc23fab553aa433f25311ed555089929d6647e451a0c3129c0ad3e2b8c0ad3","metric_details":{"best_score_across_scorers":"0.04","model_release_date":"2026-06-01","stderr":1.9694638556693238},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jyWVTGV3ZXhotEFLLGQsKi","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjyWVTGV3ZXhotEFLLGQsKi.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/jyWVTGV3ZXhotEFLLGQsKi.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"98ebd1b9a432cfc30e1b23b7954a08307c40c4040fed7df2a78f8978bec2ace1","result":{"confidence_high":7.8601491571118745,"confidence_low":0.13985084288812555,"sample_count":100,"score":4.0,"score_display":"4.0","source_stated_rank":169},"run_fingerprint":"8f6afca1211cdb66cf8005d640c1ab11024c2f1bffdf7d148f3cad0f419af972","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e15386937757ec3e2088b88f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T17:45:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T17:45:57.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"963f65a3531d68d306692cacdcf44a18f10082f113218ce3ac6e7b04c3422aef","metric_details":{"best_score_across_scorers":"0.14","model_release_date":"2026-06-01","stderr":3.4873508801977695},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"XwemU6ABotEuG9o5TRLG2r","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXwemU6ABotEuG9o5TRLG2r.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/XwemU6ABotEuG9o5TRLG2r.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0a00007879a27d0310840dd4a593701e5f20382a2112a1bbbf97a4626268901e","result":{"confidence_high":20.83520772518763,"confidence_low":7.164792274812374,"sample_count":100,"score":14.000000000000002,"score_display":"14.0","source_stated_rank":113},"run_fingerprint":"f9d14d2b1b4a74de3b9562a6b27299540df8fcecdb2069638dad72bf17523d96","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a7cb6ae0439d74319c328f69","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":125,"at":"2026-06-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-02","status":"stale"},"measurement_id":"2f8518287a4f9e294ae6b1cba00cc41c7b773eeb791a77227121191792090028","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run; vendor scores using a different agent or tool setup remain separate supplemental runs.","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","cost_per_test_usd":0.416355,"date_is_proxy":false,"latency_seconds":726.041,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/minimax_MiniMax-M3","source_locator":"Results · Overall accuracy","stderr":1.938},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-uniform-vs-vendor-supplemental","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":512000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"07c6327981f5ae3edc71a63881e3479f5aa7c770ba6ced42003fc435e75d64eb","temperature":1,"top_p":0.95,"upstream_model_id":"minimax/MiniMax-M3"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"45aad32b6aee80b16a6f1985568b4f8d87337b23cb5c951f81df43d2d7022825","result":{"confidence_high":78.79848,"confidence_low":71.20152,"sample_count":500,"score":75.0,"score_display":"75.0","source_stated_rank":44},"run_fingerprint":"3dc07c332263d21fa194f9eaf0fdfa809aac480459c524a7174750e496da0641","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_02c909f667465bb33109fcad","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8945dc6dddb53f0ab9024730c0cd81936de3b1c5ba4b3c04f660ed32c8e26ba4","metric_details":{"model_release_date":"2026-05-31","pass_at_5":69.36936936936937,"potential_contamination":true,"sem":1.125224864576288},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"contamination-risk","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"MiniMax M3__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"9b03de353448bec4c09d54b83315b7951e30c2d8dba467c278068f98a4886798","result":{"confidence_high":49.412647941776726,"confidence_low":45.001766472637684,"sample_count":null,"score":47.207207207207205,"score_display":"47.21","source_stated_rank":7},"run_fingerprint":"0a4e559414c29b68877d607a5553e35ebd2551fc5ad5f53dd3ba808cdfc143a6","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_da68402d6b49d63678a6cd3c","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"contamination-risk","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_8d9de222c7fa49c448653fc0","split_or_window":"2026-06-01/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-29T23:53:02Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"2d09ca24522f02b1b521f6345c96a34b7c64ecd2a008369a620c5cf4764671f8","metric_details":{"comparison_warning":"Official post-release SWE-rebench window; use only for same-window comparisons and never mix it into the canonical rolling-window rank.","comparison_warning_code":"swe-rebench-post-release-window","model_release_date":"2026-05-31","pass_at_5":64.28571428571429,"potential_contamination":false,"sem":3.57935628056498},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-post-release-window","comparison_warning_code":"swe-rebench-post-release-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","supplements_range_key":"1778803200000:1782864000000","upstream_model_id":"MiniMax M3__tools","window_from":"2026-06-01","window_status":"supplemental-post-release","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"d0d9c6e1df1f0683aebf39e74c887fc280d9025e54612c0cc9fa0e3a85e6bd73","result":{"confidence_high":50.82506211943118,"confidence_low":36.793985499616454,"sample_count":null,"score":43.80952380952382,"score_display":"43.81","source_stated_rank":null},"run_fingerprint":"264bccba2608f656ea08767961d15b07a044672bd709032331dcd408bd2efdc7","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_da68402d6b49d63678a6cd3c","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-post-release-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_fc9656f6e770f0821df2c3d6","split_or_window":"vendor-public-own-harness","unit":"percent","version":"SWE-bench Pro · MiniMax provider run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":126,"at":"2026-06-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T19:13:32Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-01","status":"stale"},"measurement_id":"99b22067aa13f889747aeeadb370ca96999c079ef56878288fb3e90cdb6014e2","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Frontier Coding and Agentic Capabilities; SWE-Bench Pro bullet; Evaluation Methodology"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"MiniMax","evidence_verified":true,"freshness_proxy":true,"protocol":"MiniMax internal infrastructure; Claude Code scaffold; testing logic aligned with the official evaluation; exact public-task revision and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-06-01","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"adc52c2f3e5c917ed0fd4886450e6be5ac794fe2cdab9915c5df9c6c8229b378","source_published_at":"2026-06-01","tools":"terminal and code-editing tools"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"terminal and code-editing tools"},"protocol_fingerprint":"f0681a222b04cce5f1873c2d9461fe93179d8713a60b82470dc4f9afdba5a638","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.0,"score_display":"59.0","source_stated_rank":null},"run_fingerprint":"9675730b35e8478f4611d8b893b7cd6e96824432efec3e4ceef4e4a105bf45e3","source":{"content_hash":"d678134166bf4098dfd750ee1d4db60556d842cd057e4951b17e03613f6e0678","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-30T18:53:26Z","homepage_url":"https://www.minimax.io/blog/minimax-m3","id":"minimax-m3-report","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Frontier Coding and Agentic Capabilities; SWE-Bench Pro bullet; Evaluation Methodology","name":"MiniMax · M3 launch report","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.minimax.io/blog/minimax-m3"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_eb4c46e38e87a4548c65dc96","provider_config":{"source_id":"minimax-m3-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cf6ce724f37d1bb8a354e7ad5dfd814e9ff43eaeef4c179c81bfb95a80ff546b","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-01","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"MiniMax M3","source_effort":"none","source_model_version":"MiniMax-M3","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"22dbfe3fc083c8b5919e05121ea8327563fbfacbe07253034fcbf3f801d471a2","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":14.7,"score_display":"14.7","source_stated_rank":36},"run_fingerprint":"4fd96894d16be02d22034e2d908cf993aeae02f3b07dc05a3e90546de4e35a4b","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e15386937757ec3e2088b88f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"833ef49816eee2ff510400719da6caa5e3c00eb5b4626740bbf7ec794f4781ee","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.03743,"date_is_proxy":true,"latency_seconds":366.506,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.055},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":512000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":0.95,"upstream_model_id":"minimax/MiniMax-M3"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"11bf83b7a7ce310e04d19d56af6c7910e47f6a52c8c5a29e922b6e5f619ee80b","result":{"confidence_high":84.2218,"confidence_low":80.08619999999999,"sample_count":null,"score":82.154,"score_display":"82.2","source_stated_rank":56},"run_fingerprint":"506da14333a331d06b6e646255088549f67fb1db3e1f19456588738cdb9de392","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ef51fefcba684ed7611f3d0c","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9b8de8c0ff4821ca1fa9c3cee273113bc2d23fe78960887ae5fc3f6502883b0d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-01","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"MiniMax-M3","source_effort":null,"source_model_version":"MiniMax-M3","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ef33c85a77b01f2b6c6767d505eb57e69b4cbf6c39ccdce61e2de8153203dbb2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":47.1064814814815,"score_display":"47.1","source_stated_rank":110},"run_fingerprint":"fa442333409b124ff4da02b3dd20969784185c022d4e0d1ff59d332fa4770027","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_09b0d8ecd750138e32e398a3","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"74c73894f1bdffee438614004e658285f58d44e7fd461090e46bc41078cd0d27","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · MiniMax-M3 · scicode"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"minimax-m3","upstream_model_label":"MiniMax-M3"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"2e090d2a3687cbde0ce3f1e1afadbca3093aaea2941a1b7a63d4f39b8a8d164b","result":{"confidence_high":52.871654,"confidence_low":41.417484,"sample_count":288,"score":47.106481,"score_display":"47.1","source_stated_rank":null},"run_fingerprint":"269655e5f11a26f5924b873dbc5af2862e2b259cd94a91205b5d85b8789e14c7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiniMax-M3 · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18bf23dd51a56ac5e133a7c1","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3c2d5752a25630373846f2214af9a7a0487fa9ee3455a960653c4a79adb83cdc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.111764","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-01","notes":null,"upstream_source_url":null},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"minimax-m3","source_effort":null,"source_model_version":"MiniMax-M3","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"8b1229a65e9d84d7833714acd10b9efea2d02da2e4034328743cfdc9dbfc1d64","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":640.02,"score_display":"640.02","source_stated_rank":86},"run_fingerprint":"1d4ecf4da724e692bbaa0546fbf9220146acfe0e436aff90024def7ae3ec85cd","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bb51e947973ce8b8932407ec","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"71045a7d0b0da4d8707c4e9f74bfb993c85706cb9185806f179ed303fc8f8941","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":6.453899,"date_is_proxy":true,"latency_seconds":5073.23,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.442},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":512000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"minimax/MiniMax-M3"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"d40c0dba471b1fcc8e381880fd2285478998ab5e0bac1fffcb7bfbee2472b191","result":{"confidence_high":58.23232,"confidence_low":36.899680000000004,"sample_count":null,"score":47.566,"score_display":"47.6","source_stated_rank":57},"run_fingerprint":"10f049c28114b650698a349528ef4bf138a6924f0c76e26cd414b1f62db001ea","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20bf9dd5ef508261c42af9c9","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"04a2685bc72b460dff1216bc6fb5e6c46bbce9e364bb4e8146403c24d69c4dfb","metric_details":{"license":"MiniMax Community License","variance":8.39673893098498},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"minimax-m3","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"22b6eeac8d30582ffd5e8747eecbca679d5e56b8a14c11774d0197f4f9e44122","result":{"confidence_high":1487.715575811352,"confidence_low":1476.3567507257194,"sample_count":17613,"score":1482.0361632685358,"score_display":"1482","source_stated_rank":58},"run_fingerprint":"1422b71dd537e3dce6b12bd9dbce2e0fb99a997673532a6118a762bc1ee7b55d","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_565e2d3f37afb7a4e6e7b33b","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1fbc166fb205022e005681d12e18e0606a6f9d84f8acfb3849ebcf1279d21be0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-01","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"MiniMax-M3","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"bd6b7d6965862d9a15394365ddd5825522232e517eb378cf9788a28723e65bc3","result":{"confidence_high":1493.15,"confidence_low":1480.25,"sample_count":12686,"score":1486.7,"score_display":"1486.70","source_stated_rank":56},"run_fingerprint":"e86f82e6a9c9e13913cfcb017da17c5418fa667775e19b465e4d9e7cf9884658","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9364e61f69b7165b3f9b8978","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":125,"at":"2026-06-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-06-02","status":"stale"},"measurement_id":"43ee83e80accf8fdf00e76218e67a872769614f62878af660804f756986a77ea","metric_details":{"benchmark_page_updated":"2026-09-28","cost_per_test_usd":0.205973,"date_is_proxy":false,"latency_seconds":1023.416,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/minimax_MiniMax-M3","source_locator":"Results · Overall accuracy","stderr":0.749},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":512000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"4835e72b2cdb794e5cf553121e3cb0644ec880033bc939b43178f60c20dd3fdb","temperature":1,"top_p":0.95,"upstream_model_id":"minimax/MiniMax-M3"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"10b86bfe47ac0edf71edf7dba9c50662f35317035b8fe9faf2aa8e9d07e05f09","result":{"confidence_high":55.02604,"confidence_low":52.08996,"sample_count":null,"score":53.558,"score_display":"53.6","source_stated_rank":54},"run_fingerprint":"260cf3a97d7eeb2d6752074666c948f9b91ea628dc76907d9183fc47604ca9ea","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_abbe9505531d90fc1ec94ed9","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"078752b7a87d5047dfe360052403486b100ed8dfb7a85b3ea5d9057daa504997","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-01","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiniMax-M3 · terminalbenchV21"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"minimax-m3","upstream_model_label":"MiniMax-M3"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"69f59dde804badedc07f95d6e4b81118d5343aa04c47c2209be53a15ac6cb6da","result":{"confidence_high":74.252667,"confidence_low":54.829124,"sample_count":89,"score":65.168539,"score_display":"65.2","source_stated_rank":null},"run_fingerprint":"bdd92715fe698d27164c032e07e63fe6a807a9523f46663c519f9c7497fbed78","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiniMax-M3 · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18bf23dd51a56ac5e133a7c1","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_bde217160cbff9a7aeb3cce7","split_or_window":"terminus-2","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":126,"at":"2026-06-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T19:13:32Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-01","status":"stale"},"measurement_id":"f2146ae5706fb0379a544acb7738b08bc94433cd948fc7c35f24fef8870a9b99","metric_details":{"comparison_warning":"Different harness and infrastructure from the mini-SWE-agent/GKE runs.","comparison_warning_code":"terminal-different-harness-infrastructure","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Frontier Coding and Agentic Capabilities; Terminal-Bench 2.1 bullet; Evaluation Methodology"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"terminal-different-harness-infrastructure","judge":null,"run_config":{"curated_transcription":true,"evaluator":"MiniMax","evidence_verified":true,"freshness_proxy":true,"protocol":"MiniMax internal infrastructure; 8C16G sandbox; two-hour timeout; 128K max output tokens; Terminus 2 scaffold; official API; exact run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-06-01","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"adc52c2f3e5c917ed0fd4886450e6be5ac794fe2cdab9915c5df9c6c8229b378","source_published_at":"2026-06-01","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"38f0bb6bc865561c737f563ea5ad90f49e1dffc542ca0546972046390e5ec856","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.0,"score_display":"66.0","source_stated_rank":null},"run_fingerprint":"cb0a7c8650c75a3dc13e1c482efb1ad50190b04a04d9c2e6619159ca9397d9fe","source":{"content_hash":"d678134166bf4098dfd750ee1d4db60556d842cd057e4951b17e03613f6e0678","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-30T18:53:26Z","homepage_url":"https://www.minimax.io/blog/minimax-m3","id":"minimax-m3-report","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Frontier Coding and Agentic Capabilities; Terminal-Bench 2.1 bullet; Evaluation Methodology","name":"MiniMax · M3 launch report","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.minimax.io/blog/minimax-m3"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_eb4c46e38e87a4548c65dc96","provider_config":{"source_id":"minimax-m3-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5cfd364e025962a3049912ae0a174e94c85cf3b73e4799a341bd988a635cc568","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · MiniMax-M3 · tauBanking"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"minimax-m3","upstream_model_label":"MiniMax-M3"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"e5cd6df65d5edef8507fb4a7f62f70f0e461eaa0b341ca3c0c5922b1a795bb5e","result":{"confidence_high":23.723247,"confidence_low":9.439258,"sample_count":97,"score":15.257732,"score_display":"15.3","source_stated_rank":null},"run_fingerprint":"8c9deb569fc905280ddef9aa55e951fe9e228a39977bce329c6eacb1cc10147a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiniMax-M3 · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18bf23dd51a56ac5e133a7c1","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"accuracy","name":"MCP-Atlas · Scale","release_id":"release_4873e5f09254ceba3d71ac2e","split_or_window":"provider-official-code-public-set","unit":"percent","version":"MCP-Atlas Public Set"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":126,"at":"2026-06-01","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-08-31T19:13:32Z","last_confirmed_at":"2026-10-05T12:33:37Z","observed_at":"2026-10-05T12:33:37Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-01","status":"stale"},"measurement_id":"c3b8638d07a0509d6e894ac2e87eb110c56e4bc32b828b9f8ecb1b85ee299b60","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Frontier Coding and Agentic Capabilities; MCP Atlas bullet; Evaluation Methodology"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":"Gemini 2.5 Pro","run_config":{"curated_transcription":true,"evaluator":"MiniMax","evidence_verified":true,"freshness_proxy":true,"protocol":"Official MCP-Atlas codebase; Public Set; Gemini 2.5 Pro scoring model aligned with the official evaluator; exact harness and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-06-01","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"adc52c2f3e5c917ed0fd4886450e6be5ac794fe2cdab9915c5df9c6c8229b378","source_published_at":"2026-06-01","tools":"MCP tool servers"},"run_count":null,"scaffold":"MiniMax provider harness not disclosed","tools_allowed":"MCP tool servers"},"protocol_fingerprint":"692429c51c6331edf0093d5c11e985ea1aed6e3be000fbc40aff45305a739018","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.2,"score_display":"74.2","source_stated_rank":null},"run_fingerprint":"487527c93fb4d317473876af797d7e72dc9ffddd02ca732911cbc56109a69096","source":{"content_hash":"d678134166bf4098dfd750ee1d4db60556d842cd057e4951b17e03613f6e0678","fetched_at":"2026-10-05T12:33:37Z","first_fetched_at":"2026-09-30T18:53:26Z","homepage_url":"https://www.minimax.io/blog/minimax-m3","id":"minimax-m3-report","last_fetched_at":"2026-10-05T12:33:37Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Frontier Coding and Agentic Capabilities; MCP Atlas bullet; Evaluation Methodology","name":"MiniMax · M3 launch report","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.minimax.io/blog/minimax-m3"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_eb4c46e38e87a4548c65dc96","provider_config":{"source_id":"minimax-m3-report","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"42d0b77222d0d07636ff0449edee1b0ecf9181377d55db26755faa11f290bd09","metric_details":{"expected_trials_per_mode":261,"normalized_gain":33.2,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":23.3},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":61.8,"confidence_low":44.2,"sample_count":87,"score":53.0,"score_display":"53.0","source_stated_rank":10},"run_fingerprint":"a00f5a464fa7216792c4963634252e872ba0655a026193ceb678b311a2241752","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_09be712f9836bbf7d088767a","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"4feb564b01c9809fd5496c141cb2e2e175dca9b1557f2a33240c482290559373","metric_details":{"expected_trials_per_mode":261,"normalized_gain":33.2,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":23.3},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":37.4,"confidence_low":22.0,"sample_count":87,"score":29.7,"score_display":"29.7","source_stated_rank":14},"run_fingerprint":"0af533498b09adaddec0bac9ea999f10afa1fa3f92d5bc5f956ec2d66850de50","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_09be712f9836bbf7d088767a","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2afacdc1118e1d7ad347df4f2da52174eaf8a28d167798ce0cd0fee723715e09","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.535","mean_standard_error":"4.5","model_release_date":"2026-06-01","notes":null,"standard_error_points":490.00000000000006,"upstream_source_url":null},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"MiniMax-M3","source_effort":null,"source_model_version":"MiniMax-M3","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"227f228996b0ffad2cfecaa2cd5cf238fcd51a5fc5f16cc5f58f45016e30d65a","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":37.7,"score_display":"37.7","source_stated_rank":33},"run_fingerprint":"d8cc961b7f504f2afa54bd2e84c65641b0abb8af534e09119116152ef2f0144f","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_53416c63888b302d0224ceb0","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5eaa6144c9d9d19b59c44f9650f7d626d344998bca31f51e020c2945325366c8","metric_details":{"confidence_level":0.95,"license":"MiniMax Community License","session_count":41267},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Minimax M3","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b09e78333a6bcda361f5417e82c3021e87ffcc85c944e6ccf662d313fccfbab3","result":{"confidence_high":-0.05992493455767178,"confidence_low":-0.0775638626097857,"sample_count":3338791,"score":-0.06874439858372874,"score_display":"-0.0687","source_stated_rank":42},"run_fingerprint":"e33b9c34d88b63a8d6bca4113ce7b8c8bdfbdca6a22e59d24e4413967ae5c147","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b5748244f086cfcc6a37e5d2","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b7661aa92aed9da4a5c91f70278ff4d29376ce56d7b3e30a8c99cab2afcee267","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-01","notes":null,"upstream_source_url":null},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"MiniMax-M3","source_effort":null,"source_model_version":"MiniMax-M3","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"de681cc903cad707fbf63a3cf8ceb5e6ebb6489f2344af45c1ea84c947480979","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2157.7716666666665,"score_display":"2157.77","source_stated_rank":46},"run_fingerprint":"3fd9985923e69d3edc9a6e092e4c9a3e278d502dae8f9b222a007a676bbf40e3","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_951eac4e039a4dc0d2cf0bf1","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"73391427eb4e25f43c46f60c814c82aa88b0f2f2bc6ab59319b8da4e797c658c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-06-01","reported_confidence_interval":"-18 / +18","source_locator":"Leaderboard models table · MiniMax-M3 · gdpval"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"minimax-m3","upstream_model_label":"MiniMax-M3"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"98cc585e975036b85961aeac40b44db480d204e7fc4fca83e42b924d7e1fc5bc","result":{"confidence_high":1263.62,"confidence_low":1228.05,"sample_count":220,"score":1245.83,"score_display":"1246","source_stated_rank":null},"run_fingerprint":"f99ba00ccfef4b3a03e1eb4c23eb0821a3b2b42ddfd4efcfb3b53a5b8426a326","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiniMax-M3 · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18bf23dd51a56ac5e133a7c1","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e5ff357a54e543ef51890cafc7bed9d99b7143cfd22381d4795c5731f139fc62","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-01","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"minimax/minimax-m3","name":"MiniMax M3","provider":"MiniMax","weights_status":"unknown"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"MiniMax M3 (enabled)","source_effort":"enabled","source_model_version":"MiniMax-M3","source_row_date":null,"source_scaffold":"standard","upstream_leaderboard_url":null},"run_count":null,"scaffold":"standard","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"fa4b66d5a1199a067fe5b1925d4b19605dfcddffa329b61b71c74d6d20670efc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4.6,"score_display":"4.6","source_stated_rank":14},"run_fingerprint":"d1e160ad1537f02ae56062ffd373d6ac38e3ec1f05a6afceae9d1ab24eb939b5","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_91769ced1b80cf6db8086b91","provider_config":{"source_id":"epoch-osworld-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"978d2de1221d5b4b57d17f84c596afd7a8e77f6e10bc27fdff2b9ee0bd6eae2c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Max) · intelligenceIndex"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna","upstream_model_label":"GPT-6 Luna (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"e2f7d3e03808b505f5497284a9dcd303a2acd99faef280a510618068575e9680","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":38.124519,"score_display":"38.1","source_stated_rank":null},"run_fingerprint":"ef37cb322d9656c650b2e2ab36a6f101d99f220171e6da5966328d138158b013","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Max) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"20f6fab3a0345c726217f90c14eb304fc596db40d3f92d317342f89ac8db6bbe","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · intelligenceIndex"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-xhigh","upstream_model_label":"GPT-6 Luna (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"c1e15747ea4b37509cd068d66419c17eb06dfa1a4916238dccd6c8dc820872f4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.558732,"score_display":"34.6","source_stated_rank":null},"run_fingerprint":"0899ac1cdede5b67ff4dca85885052580313ff9cca758cb0b5c76fefefc11c01","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"728950d4053a834c94bb5c94dc85b7887b052b70bbb1cfbb0195e98f4ef6e4da","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (High) · intelligenceIndex"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-high","upstream_model_label":"GPT-6 Luna (High)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"79b52af1049e6912724426b227d2acf23b4b38bc07187209c8acd75035e2ce4c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.928205,"score_display":"32.9","source_stated_rank":null},"run_fingerprint":"83ce5aee68f9c48cb2df4a6c3c2b678f905a96143950e917086d3e238b0c886c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (High) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"61ec34104091c27c32fc3c9573ba4260002f60c0f0b44ceeb39df3dec70b5e29","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Medium) · intelligenceIndex"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-medium","upstream_model_label":"GPT-6 Luna (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"5ff33c569e1af6b5c3672d5e67c257134c265842b3defb8b54ba26701b1fcbf5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.925177,"score_display":"29.9","source_stated_rank":null},"run_fingerprint":"b992311da666c297f01704ebee0802419e637171a8bc7404426b8aa2ed18f381","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Medium) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"64354ce5380384f18fd6ad68cae1ee913496463b9eb121bc8b986be1b20a5390","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Low) · intelligenceIndex"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-low","upstream_model_label":"GPT-6 Luna (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"65ec0838cf305d291173f416c73a6a9440c2712e6a83d5392708809280751203","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":21.526198,"score_display":"21.5","source_stated_rank":null},"run_fingerprint":"0ce0749c12e6e5fa1647d39698d28255f131b7e03a58cc318626c891b2ce6fef","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Low) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5e4abd1baf6cb696527546469979fde1f8f2edd38282d9df70523f9fb334a7db","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · intelligenceIndex"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-non-reasoning","upstream_model_label":"GPT-6 Luna (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"85a5f6db353ca710667964e10cb19914a4e108cca3a22c9afa739ba3d0167d59","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.467847,"score_display":"18.5","source_stated_rank":null},"run_fingerprint":"e7b2d9ea12030eebb7d22783839e3d8b68ffcea01f44e12c79813a71700df746","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7c03858cd0e0f3651464231f4fe34364fe8b149ba82037e46d7bf2d50ce1fcf3","metric_details":{"license":"Proprietary","variance":11.118023623641653},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1449.4154042382609,"confidence_low":1436.3449138239384,"sample_count":10014,"score":1442.8801590310995,"score_display":"1443","source_stated_rank":90},"run_fingerprint":"836b94c779366df0e592eed725918f83dc9100c7fc4ccfeaa78d4c4901702658","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5502ab3c5925abcfc556e2c9eacd92b39c3f5b5beab8a1de86093b3c1d1732ff","metric_details":{"category_scores":{"Agentic Coding":51.211999999999996,"Coding":78.95,"Data Analysis":73.36966666666667,"IF":55.93325,"Language":73.82766666666667,"Mathematics":89.1235,"Reasoning":81.76925}},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.02647619047619,"score_display":"72.03","source_stated_rank":50},"run_fingerprint":"437d668acec6d2df40dfdc0e08dfb7c7005d1adc8f33f3cad1d2015a3f371bc3","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6fd8c730cf656e6d46c64f991d4ec8f0f0f4886047a1163902fe4829fb5773af","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Max) · hle"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna","upstream_model_label":"GPT-6 Luna (Max)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a640db3608f4eb02c8550cb39d30f1ebda9cef9471d64e28bbbf55b51b535d50","result":{"confidence_high":40.579699,"confidence_low":36.4769,"sample_count":2158,"score":38.507878,"score_display":"38.5","source_stated_rank":null},"run_fingerprint":"80bc932bd3fba60dcdb1898e8710ab6c78befc0033bb6738ebf5c73b4fee9842","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Max) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"841f5908a1fa1833bf8c0ba0d47c887c59e290c13f984e32d6150bc2975ad4dd","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · hle"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-xhigh","upstream_model_label":"GPT-6 Luna (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"dd9e101978f83057f645a49ce7e51b5efc52d2054fb98e058af9ba992acaa4d0","result":{"confidence_high":36.367019,"confidence_low":32.363345,"sample_count":2158,"score":34.337349,"score_display":"34.3","source_stated_rank":null},"run_fingerprint":"a9d6dfb909b5198279282c905a8a49605cdb0d5d78a752fa075599ede181d960","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e6d1a75530dda58a7c1b193857cbc11565f64df818614640d99b757ae729e6dd","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (High) · hle"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-high","upstream_model_label":"GPT-6 Luna (High)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a5689ad7c426f09aea060fee64b8f12580ebb2fccdc65b5450d9478d686c897f","result":{"confidence_high":34.95906,"confidence_low":30.995892,"sample_count":2158,"score":32.947173,"score_display":"32.9","source_stated_rank":null},"run_fingerprint":"941c240f0a0adabeafb0ae522ea15e5e09ea770016046568a7cdfc4847c605cc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (High) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"81ab9749be7be4ba73d070e8ceea7253b0e7082bb0ddec7a50100158e3c82450","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Medium) · hle"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-medium","upstream_model_label":"GPT-6 Luna (Medium)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a10b6d95bd590b3ce706e57ffdf19c91b92a1634bd015979bfa543ac2bc55657","result":{"confidence_high":30.204132,"confidence_low":26.406936,"sample_count":2158,"score":28.266914,"score_display":"28.3","source_stated_rank":null},"run_fingerprint":"56a11d3b1bcf8f15ed9f925401c6cc132ae95026c85326580b34e8a8b3890ae4","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Medium) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4fcec938d5bd80a7790a13a789985ca248d7086910ad52566af0edfacef9d1bd","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Low) · hle"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-low","upstream_model_label":"GPT-6 Luna (Low)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"4c88c3ecd9dd8e9c9f54b3b6d9464208fc7f2b92e58fd0a982d0462957a1b293","result":{"confidence_high":21.997963,"confidence_low":18.608232,"sample_count":2158,"score":20.250232,"score_display":"20.3","source_stated_rank":null},"run_fingerprint":"cd2b8bfe5b1ac186d66caf25b435785d0b3f4afad22885e9bbffe4867940c974","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Low) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"def306dd419a087df11ba3cf39048f11b1ebee3de274233a077489abeb21de53","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · hle"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-non-reasoning","upstream_model_label":"GPT-6 Luna (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0a545f10c4adb217622fcbe8184b2339e1c083af4c040314e201dd469b6ffa4d","result":{"confidence_high":9.828826,"confidence_low":7.463912,"sample_count":2158,"score":8.572753,"score_display":"8.6","source_stated_rank":null},"run_fingerprint":"f6b3459806aa1ae2331aa56b484112d33fd4e8f9fdcf06d71e3ac495711dd8f2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T22:14:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T22:14:05.000Z","first_observed_at":"2026-10-01T00:45:05Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5ba1ed6d0614e196e00630d3da738f4e3643ecc888340e159dc04111f9b2c33a","metric_details":{"best_score_across_scorers":"0.414","model_release_date":"2026-09-22","stderr":1.5583544104177554},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"HX9sRVWJtZiNvneALnVzs8","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"e94db4081e6470cebd76cf723afdbee751fd5ab78aa7caf1dce8051428180c7c","result":{"confidence_high":44.4543746444188,"confidence_low":38.3456253555812,"sample_count":1000,"score":41.4,"score_display":"41.4","source_stated_rank":42},"run_fingerprint":"f7cd258aab32af05d9d492bb01c73c1848210d5e5197de2ef6f20a68c9167f09","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":12,"at":"2026-09-22T22:14:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T22:14:05.000Z","first_observed_at":"2026-10-01T00:45:04Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"42351ef86fee07e1fe2803f6a39d038c6d6daef8b8e37c50214b1e432db38679","metric_details":{"best_score_across_scorers":"0.9046717171717171","model_release_date":"2026-09-22","stderr":1.8537378631915502},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7F3QFMAkwnCV62HcvxP96q","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"4f8894782fe7146bdd6e7823b91e16cebd928ac74a010334a517c63a28e2063d","result":{"confidence_high":94.10049792902714,"confidence_low":86.83384550531628,"sample_count":null,"score":90.46717171717171,"score_display":"90.5","source_stated_rank":43},"run_fingerprint":"adb0d3742ed6d32db3165c81d18ccc69f3a9ed9ef6baa68e708c6a406b1ca0f2","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T22:14:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T22:14:05.000Z","first_observed_at":"2026-10-01T00:44:54Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"83ae77366476286ba2b5e890ee33e86c402e6fa6bd27e2a47db594f66b884206","metric_details":{"best_score_across_scorers":"0.9888888888888889","model_release_date":"2026-09-22","stderr":0.6674027922425511},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4uDz9b6SosxDSZf7swMD9D","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f54e9344c5dcccd9c9099f125e8e3dfabc47aca4dc7b6a2376d820bcd2bc5cde","result":{"confidence_high":100.0,"confidence_low":97.58077941609349,"sample_count":45,"score":98.88888888888889,"score_display":"98.9","source_stated_rank":17},"run_fingerprint":"dc4b580d58cd8e8e4db22cc9a27a23ed064e349b628946d4cb45ecc31c645ba2","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T22:14:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T22:14:05.000Z","first_observed_at":"2026-10-01T00:44:54Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e12c60c9c739c56a51c114ed00d4ddb5d97a5127a8ae5960ca9110605c923832","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.7894736842105263","leaderboard_private_problem_count":285,"model_release_date":"2026-09-22","public_example_count":10,"stderr":2.4191503146153774},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"iBQxysYR7zPxB6hxvJyTLc","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Luna (max)","thinking":null,"upstream_model_id":"gpt-6-luna_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"afddc02d7d2d96b25d4a0797f0a91eb3d6acc65cb504e2ac84997901fb605d67","result":{"confidence_high":83.68890303769876,"confidence_low":74.2058338044065,"sample_count":285,"score":78.94736842105263,"score_display":"78.9","source_stated_rank":16},"run_fingerprint":"82eb6d545ba17d1da2b1c4081d32a588bafa1ad25fd3052a635852e122238443","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1b0255a51c800fc56288366ca6f5a5e0993b4ebf4e9dfe9def275852fbcfdddc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.06181100083333334,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-luna"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-6-luna-max"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"acc7fae2580193dd8d8dda87d8933ecd7f1fc199f9fbe30a6a65459cc2f0fa72","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.30555555555556,"score_display":"59","source_stated_rank":82},"run_fingerprint":"4fd282ab354f230ec4ae5b7942862fb59c39b7d2760080cc762925f045a51661","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"35304e6dfd314da7f4b7f76145c2d244c59c58372c5106e28304f24ae67e8c02","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.032297893333333334,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-luna"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-6-luna-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"5124c24ac9696e9354b9bb897db90ff3f3820f699e77e7d3dd864a42e5ecfb42","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.94444444444444,"score_display":"42","source_stated_rank":104},"run_fingerprint":"cecd16c1e8b6d47f0772ad304125c5e3132e8890b00076310eb5dc1d29051cbb","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cf2442aa66e9a3b235fe768ff82c0a05743d66f5c8da402906d19b91794e0711","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.02336191,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-luna"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-6-luna-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"85205148742eca947c240ec07abb751a452c47c3372d53bc06be85e45dded213","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.38888888888889,"score_display":"31","source_stated_rank":116},"run_fingerprint":"8cc9503c6aba4445471f9a3e3dc89022f2e7594819c08e7646334991a0b5469e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ab66c7bb701d22addb8f4a95b87ea0f13596bad1ffd6b734c87da0e21f9ea326","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.012202668333333333,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-luna"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-6-luna-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"108600b649a8da13a163d30c461afdab6f7dc44bb2c192fb4306f5f319dbdeb6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.055555555555554,"score_display":"18","source_stated_rank":132},"run_fingerprint":"789d1c8391783a945e175fd5fd06d39afb131f491164d04f3fb00b663c18abec","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"16d0ae09a58b491b84abc25eacbabe7f3cc2d63306a2b66d69cda80f0381112f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.005273818333333334,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-luna"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-6-luna-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cf9e1cf380934a842281701eb2907f356b0c1f0d607bc33e38a3842154942939","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4.583333333333333,"score_display":"5","source_stated_rank":169},"run_fingerprint":"63994f85b5ae0135580ff0489106413df4fc7f73eb5bb6c02b1f52d84612fc1b","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:20Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:20Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"78f63a9ff365958aa8af6d8376e747f725f51c06bc60b36cebee4eba09c8ab3b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0036990350000000003,"model_release_date":"2026-09-22T00:00:00.000Z","results_url":"https://arcprize.org/results/openai-gpt-6-luna"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"openai-gpt-6-luna","model_type":"CoT","upstream_model_id":"openai-gpt-6-luna-none"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f8e69cd1524ad0e0994ec3a52e26bb8dcda0f1a6ebdc90e723087d8f470e5cf9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.0,"score_display":"0","source_stated_rank":239},"run_fingerprint":"a799e8cf56d1dc853b89cc570638b4452357d57dfe2ca74c40bb7711445a96a8","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"74536c01e2261f2cd761a305533f13adfcd90f8d6b62a858b55d21e9983973ea","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (Max)","source_effort":null,"source_model_version":"gpt-6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"985c3341bedff0348e58553a0baeffab39e649d9f023c4914628e3dcc1089ea0","result":{"confidence_high":64.256211,"confidence_low":54.158396,"sample_count":360,"score":59.30555555555556,"score_display":"59.3","source_stated_rank":84},"run_fingerprint":"6273935306894f68c0d22721a4492052c13a6dad4f43122685dc2859537c3afd","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a1b308fa3bdc364c1d9d1a5eecdc3e596a1f714de455fe21b5e38faf3e116fe8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (XHigh)","source_effort":null,"source_model_version":"gpt-6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"61e3aff7abb8ef87bdceaffba4c6d59e2fa0eec594c11473719b0cedfc8f4c64","result":{"confidence_high":47.100808,"confidence_low":36.958189,"sample_count":360,"score":41.94444444444444,"score_display":"41.9","source_stated_rank":105},"run_fingerprint":"1c7dde0f009dcacecac15652f573a7cead76bbb5baaab36dea0e9643d938eaa3","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"30d02ae8c9afc8da88136660ad250501a1ce6317783617bdd49e937fc6cd715e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (High)","source_effort":null,"source_model_version":"gpt-6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"38006c3e609fc1cd3e85d85ce5bae7a0df1bb1c4ed503d364beb84a544807417","result":{"confidence_high":36.357977,"confidence_low":26.81281,"sample_count":360,"score":31.38888888888889,"score_display":"31.4","source_stated_rank":116},"run_fingerprint":"fa0cc8ff744e6f64da103140e62a5ccbc50767d46835551675e1c38907712c0c","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1c2f1b91be1c35d415bf2176a9b4532a3f61ef4389c7fecedda3e763e4e2bf1f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (Medium)","source_effort":null,"source_model_version":"gpt-6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"e147d434abd45993362802e2f1abd0eb65cfa67a957abc20de631ed5ed1bf04e","result":{"confidence_high":22.359642,"confidence_low":14.426036,"sample_count":360,"score":18.055555555555554,"score_display":"18.1","source_stated_rank":129},"run_fingerprint":"ed5ebf5f1f0f3d46de4cc68541f1ffac8592971ec52de0a37be596ecfe29b2e8","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"68ccf92656ca6d24fae9fad772d4daae8f37c89e67a668aff6ed53e2ea68c93b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (Low)","source_effort":null,"source_model_version":"gpt-6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"4b1c5880b39dd11340fa937cd7c722f33ee84f4fd1f41ee9b524c021de1495e4","result":{"confidence_high":7.264551,"confidence_low":2.861174,"sample_count":360,"score":4.583333333333333,"score_display":"4.6","source_stated_rank":165},"run_fingerprint":"07aee5f637a40005a97f3e3f92a055bf1731cb77aac9c330297ae1b8d61c37ec","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ace0322926fc3022a899a3b82e8be14cb569801d4b96834cb0c459a924d85736","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (None)","source_effort":null,"source_model_version":"gpt-6-luna_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"476f2890c2464babc900f9e5cbb23fc9f86adaeb1d73c6e9ba07afb4068bcf5f","result":{"confidence_high":1.055844,"confidence_low":0.0,"sample_count":360,"score":0.0,"score_display":"0.0","source_stated_rank":230},"run_fingerprint":"67dd745af440389299a672d164273041d1548fe07953dc62816d4653c9ff42c8","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T22:14:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T22:14:05.000Z","first_observed_at":"2026-10-01T00:44:55Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9ecbed8213913b049cd092e125646d9b0302e7a2b73b28fc1658241b4197e39b","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.5609756097560976","leaderboard_private_problem_count":41,"model_release_date":"2026-09-22","public_example_count":2,"stderr":7.846686801046543},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["max"],"epoch_id":"7n3HBUK684GmdT6Tkycu6m","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-6 Luna (max)","thinking":null,"upstream_model_id":"gpt-6-luna_max","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"521221b6baae86dd00715ef32c0e7fd47ce2cc18b3abe424e51a2346c132866e","result":{"confidence_high":71.47706710566098,"confidence_low":40.718054845558534,"sample_count":41,"score":56.09756097560976,"score_display":"56.1","source_stated_rank":20},"run_fingerprint":"b4dd97ab1d17bf330bc77c3132f27c251c58f5b008ca26b750132bc102f24601","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1f400f78c3cc62266f14b8066f7df01d2e5eca42a8e64031a1ce8550ff2d294a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Max) · mmmuPro"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna","upstream_model_label":"GPT-6 Luna (Max)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"18ae8feca09b3727e87e0c5a99925967449216100a3a5be9b856af52d41618e4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.653179,"score_display":"79.7","source_stated_rank":null},"run_fingerprint":"460f03dec4cd35f21be258bfc038e1cd0abd8a0403a0f4ec4a2e210b27dcfbf7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Max) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"399c4c9bba44630cee4d0658ea326dcfcd5615c98aaf8ae0262fbc7d39c8ad23","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · mmmuPro"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-xhigh","upstream_model_label":"GPT-6 Luna (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"10776f521dfa7f2327d58c52398de7171515005d0ab6d1b675f606fe3013063a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.710983,"score_display":"79.7","source_stated_rank":null},"run_fingerprint":"30a3fb2d50dfedb87d5b28a79ddcb230c6fa398264867835b74111214246485e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"84f3c0c1025b1e180a47cfdd3b00f552d5b7add2c19c5239d1604e2b8746aa2d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (High) · mmmuPro"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-high","upstream_model_label":"GPT-6 Luna (High)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"470a370ce6169d54a03537ba0123b5aef4270437822ca9b74dd5d95a305e3db0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.514451,"score_display":"77.5","source_stated_rank":null},"run_fingerprint":"9341b3a4e2eb2f53bae056c1e4614d8bd77bd9351a2711dffc4cda77312d859c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (High) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"46851c35f7fc2e7bc32dbdc2d86e5be3f7fc9274a11de121fd800b116c3e0efc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Medium) · mmmuPro"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-medium","upstream_model_label":"GPT-6 Luna (Medium)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"40585c060e7a82fa2b07f29c9a36cd3b9b8289a8131416591c7eaecfe51a3ee3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.861272,"score_display":"77.9","source_stated_rank":null},"run_fingerprint":"8a6cc7f10d54d5e483a3b38b8b545ee60d5392ea8450833e5da86de0880be60b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Medium) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"982462992838e64a49d1f31b1f78cbf916fb4a0315ce9319f94a18cf1436298a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Low) · mmmuPro"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-low","upstream_model_label":"GPT-6 Luna (Low)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5e66b850effb0c55275334416e4ad36dd9b73bdbca33b31ad3914030085b16e2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.953757,"score_display":"76.0","source_stated_rank":null},"run_fingerprint":"e53ac5a479359d1fe708f69491e72f943e7b24ea56590a5e3fa9cc6ee40293ea","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Low) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"46fe2a8d194e85689551f1998ab4ef7a2d41fba12d29081cf81241a3c41e43b3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · mmmuPro"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-non-reasoning","upstream_model_label":"GPT-6 Luna (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c25fc173b8a4b8fe2babbc86a15a1290443054d218ab84ece533a7dc635bb19c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.080925,"score_display":"62.1","source_stated_rank":null},"run_fingerprint":"43ace061cc58e9b4503f7cdaa08ee461756a5503d68cfe5040340fc6188ce8c6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"399bdc4e94427baa9fba552c158b7ca335b018c8984acf0c17e1ee778a962ae4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (max)","source_effort":null,"source_model_version":"gpt-6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a261c298092001fa62f0bf8ec2165a652213b2493d276dd42999b5d2779fed10","result":{"confidence_high":30.097982,"confidence_low":11.897608,"sample_count":71,"score":19.428571428571402,"score_display":"19.4","source_stated_rank":56},"run_fingerprint":"d4d1cdcde460ac011b3e2b75d0723051b035b9434e4d07053f725074fd7d5110","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b1c1021cda4b0d8a17411e4f94518289a247bb904c41f9c5b4768826170d89ff","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (xhigh)","source_effort":null,"source_model_version":"gpt-6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"864fdbd50c014a5b96017d6caa79a3749ec130759b2a710f109bedb1eceac77c","result":{"confidence_high":27.856248,"confidence_low":10.34466,"sample_count":71,"score":17.4285714285714,"score_display":"17.4","source_stated_rank":67},"run_fingerprint":"38bd05ea880eeba102a87acbd46c1b494517f4fa1155022779d1cf13c5261a8d","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"664cfb7c359d10d8e3674da1502c12065077f126e094eb2a7f47b02eecf92a27","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (high)","source_effort":null,"source_model_version":"gpt-6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"32b9d9d2fed3e407dfbb550e9ecd304f0af0f853c7c4b427c19cce724b0481c9","result":{"confidence_high":25.577174,"confidence_low":8.829054,"sample_count":71,"score":15.4285714285714,"score_display":"15.4","source_stated_rank":75},"run_fingerprint":"6916fdfda1e26af1739dac246ab5f6f396c2c4c76df81373c9109b1bc39cdcd5","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1deb8e389a81747f87b6a1b9d8f66d9db5e300dca0881fdd72bb39481ff8d760","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (medium)","source_effort":null,"source_model_version":"gpt-6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"48b208f7a36e94deb5a4b4208ec2089765567668b2afa7b584a7a4717c05a311","result":{"confidence_high":19.849425,"confidence_low":5.341149,"sample_count":71,"score":10.5714285714286,"score_display":"10.6","source_stated_rank":90},"run_fingerprint":"0dcade43fc028e2cadb14f85405c647086f8d8ffd480bc6efb5eb073aff94bc9","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7a03bed8231f2e8f2d9bdb91e95d98f7a15e35a83d6fc038cdab37aaba7cffe9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (low)","source_effort":null,"source_model_version":"gpt-6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a1d201886c71e2e366436dbf6416445d5495ace45e0a7cb67c3b39fd54a6821c","result":{"confidence_high":9.340258,"confidence_low":0.671592,"sample_count":71,"score":2.57142857142857,"score_display":"2.6","source_stated_rank":129},"run_fingerprint":"fab7800745e09eb1245cf48e3e844c644ea3180bce99674b7de78ab777529a92","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"60a5789c5cbe6f3ef4a123b68c9934c61bf0f7784c6df33c3f83ce6d3fe1ef9e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (Non-reasoning)","source_effort":null,"source_model_version":"gpt-6-luna_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8aa8b88a224cf421129dc346258b25fa960e919d86840512599f95fb0948b87a","result":{"confidence_high":7.127519,"confidence_low":0.173844,"sample_count":71,"score":1.1428571428571401,"score_display":"1.1","source_stated_rank":140},"run_fingerprint":"567ae321999db746cda980202ea76a303fc5ca447bedf531b0ffa6d3eba5b178","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"120f80f145ee267edc7b0d73954740c47923523cc8bba4c12526c70cbcf13fd4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Max) · critpt"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna","upstream_model_label":"GPT-6 Luna (Max)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"1f8879190f65bce36ca2daf2a9fc639c748f7a921a80f70e550b656ac137f2f3","result":{"confidence_high":30.18249,"confidence_low":11.855602,"sample_count":70,"score":19.428571,"score_display":"19.4","source_stated_rank":null},"run_fingerprint":"3bb793cbd780c5bff1435fec4de291ef7b13304aa90a9e874a036d4369eb8550","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Max) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3992b92b36970142355a9e540d4ae1df9f281442857e36e5ff56c403f50947a7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · critpt"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-xhigh","upstream_model_label":"GPT-6 Luna (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"37df7388e6135fef767b92ef34ff5baefc6ef1798a28163b4d1035cbc2620270","result":{"confidence_high":27.940154,"confidence_low":10.306037,"sample_count":70,"score":17.428571,"score_display":"17.4","source_stated_rank":null},"run_fingerprint":"f7e9ee35c9af1b48c4c7ce71d2af5a9cdf313848150ecc1549dbb154a64328d5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1d055880a004355c8e19a8ce593c863f43c71cb3fb03d2667aeac14b1e93e1bc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (High) · critpt"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-high","upstream_model_label":"GPT-6 Luna (High)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"4defd339bfc86cd61648761211f587de3b397a0786d110436518fcd21cfa50dd","result":{"confidence_high":25.660281,"confidence_low":8.79401,"sample_count":70,"score":15.428571,"score_display":"15.4","source_stated_rank":null},"run_fingerprint":"ed0444a1acda4d0836af1eb65c867e7f10d1d2f25f26aca61db070936c3e3c2b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (High) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4a86b7c5736d5d107511b13e7977fb1937e659d543dcb25d0b5c4c113c7c92e7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Medium) · critpt"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-medium","upstream_model_label":"GPT-6 Luna (Medium)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"517a2afecb8476a2319500d8f6926a1b1fb758b9fd6d9c67da8cc8d590a621c9","result":{"confidence_high":19.929636,"confidence_low":5.315755,"sample_count":70,"score":10.571429,"score_display":"10.6","source_stated_rank":null},"run_fingerprint":"0191ac739291e0d3fa0819149f6f55d3d914032e81f095257c8940b68e6e2682","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Medium) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f0169d16e25bd9d51e3e5d1e8b5f96f71d4155c084dd621a317036e51d613f82","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Low) · critpt"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-low","upstream_model_label":"GPT-6 Luna (Low)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8123ceef91ee4645c68279c080fdacb1065e5cac1612a4f6e8b2d12554e3d09b","result":{"confidence_high":9.41179,"confidence_low":0.665999,"sample_count":70,"score":2.571429,"score_display":"2.6","source_stated_rank":null},"run_fingerprint":"5e896046ffdd9e54e21b222303b7e7b52fd37bc2a532956b90786fcf5ab0fd19","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Low) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fe66b43e9cf46842eae49d879571c6d227bfbf9083f02e0215891a1f0479a30b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · critpt"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-non-reasoning","upstream_model_label":"GPT-6 Luna (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a7e242a6cbe1977ec47ef6e630fdf817d4568da67057293636cf3e633db620c0","result":{"confidence_high":7.197254,"confidence_low":0.172034,"sample_count":70,"score":1.142857,"score_display":"1.1","source_stated_rank":null},"run_fingerprint":"c23d037ceeaae48040e308f4d0205f19ed34c5380e44d3dbe0eefbbd31c2c17a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":12,"at":"2026-09-22T22:14:05.000Z","basis":"evaluation_started_at","evaluated_at":"2026-09-22T22:14:05.000Z","first_observed_at":"2026-10-01T20:29:23Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9f9c5b77b1af5d4eab47ed163b8d8e25b16949780e11c959453248777270df8f","metric_details":{"best_score_across_scorers":"0.31","model_release_date":"2026-09-22","stderr":4.648231987117317},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"YwzVUtcUukgH4VxszaKNsi","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3082ecb5217c27434c03ebae5416c907f8f18daf7a281d9e7f19d0de01b2c91f","result":{"confidence_high":40.110534694749944,"confidence_low":21.88946530525006,"sample_count":100,"score":31.0,"score_display":"31.0","source_stated_rank":48},"run_fingerprint":"df4b801cb39824477654afaa2eeabf8bcbb7a5e512947cb87b19069f69436d35","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"abe963a056f2d088fd71e03398423f5255b7d5bad83289f21ac7d555775174c2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (Max)","source_effort":null,"source_model_version":"gpt-6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"eed432f169eff7b8754f49763493b877a3d28a5bc99b079c4137be046585d784","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.66666666666667,"score_display":"86.7","source_stated_rank":92},"run_fingerprint":"fe87df2407eca5ce0385c78685f6d1963eeb9803a47269d9ece44c15547e585c","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"76a19fe3e1603c86b2710faf32c846e5fe645422f8e92a135e4840be9fb58397","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (XHigh)","source_effort":null,"source_model_version":"gpt-6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a8dc4b5347304b4141dc44a33af236b94d44a349a5a56671b338cadeb841f386","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.0,"score_display":"73.0","source_stated_rank":128},"run_fingerprint":"c04a3c709d61a457e278babd02c9fcdb1eeb6d98265c4f0a7772030f87ffba36","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e2a4a1a4e08ae67d135f2a689029345665d9317d8a79d43d77cc68c4a5a74bfc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (High)","source_effort":null,"source_model_version":"gpt-6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"153c94e5b69a57a27811cd4bfecba715f927f28017cf6328eaeeb3dc22cec490","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.33333333333333,"score_display":"70.3","source_stated_rank":134},"run_fingerprint":"cabec91e5f6fae3e4cf25714e7a3aef631930087eaa406b98f3ac7ef199c88ae","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"15584207f4e5f43d162704d356f84e8c03fc609e8b289b0209b868972a4266b6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (Medium)","source_effort":null,"source_model_version":"gpt-6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"42b34605956bfc5afe4721953f58fc7b3b6a7320176a79de7230f331c1137479","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":61.0,"score_display":"61.0","source_stated_rank":147},"run_fingerprint":"dfde19a4d30d965b7f69beae8d5fd59cfeb577d81c81ba17f19efb182f8ce592","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fc838f5c81f1bfcf9a1e4ac1a0d65aa366f016799ecafc7af4e970d269897716","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (Low)","source_effort":null,"source_model_version":"gpt-6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b54a55c3a669742ec66e438404e5b318a2b5568299bcce8e4d7b65169ff95744","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.666666666666664,"score_display":"37.7","source_stated_rank":185},"run_fingerprint":"2f3acf5720106867381df581d981bff6896f48d2b39fcb154e1e03388fc7ceb4","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7d3ed9cc9b11183973570ae5abf594e3ce436e7bab4207b7e6db993a5f6cc3fd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (None)","source_effort":null,"source_model_version":"gpt-6-luna_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"64955eb3cd0d58dac8a414b036256a0d93fa9028242359046a22d013ae6b35b1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":8.833333333333334,"score_display":"8.8","source_stated_rank":251},"run_fingerprint":"ccfdca464a66c559a845369a21f6ca347941ea96f35ffbab776f9fd2751dc721","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"02b897a43ea4a87a1dfea429b51c13135a9b190b84c2ec00217ec712968eaba3","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna","source_effort":"max","source_model_version":"gpt-6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"codex","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"0a6c6cc0cc8cd63bee36b5a54e5a6696b3e36023dab704c63cf28436f2c91adf","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":42.42,"score_display":"42.4","source_stated_rank":18},"run_fingerprint":"21d7649cfa7c50fcee75e146c5428877e6751cb88ef7967efbfa6caf3109996f","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"57e55691eacd9be3cb5e3662fb8b00a4a7cd18c33eec19a65260e1a6a4792fc5","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.199412,"date_is_proxy":true,"latency_seconds":2147.141,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":8.866},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"max","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6-luna"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"f0780aaf629b1a0fb03d045ce2271a492bba7c7f15291ac945969d3b11d5ca8b","result":{"confidence_high":72.93336,"confidence_low":38.17864,"sample_count":null,"score":55.556,"score_display":"55.6","source_stated_rank":19},"run_fingerprint":"e456adc401377facfe8b8b1005a972a32d7c3ef60d0ee2003f9d33a8d10e2605","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"af1a62a8ef8c266bb4918507fcffe662f084c2467cd0734c8024b8174910e09f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (max)","source_effort":null,"source_model_version":"gpt-6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0f7d4c3a3295a0e86bb7c4f50dfda01a494bc5661a0cd9cd1ed98ca1058977df","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.6296296296296,"score_display":"54.6","source_stated_rank":47},"run_fingerprint":"68cf2fdec930ec4ce4d00e6b64a31e42ae9616df37e255cdac5ee833ed90c164","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e97caa84b3bf02f986507903a78eb06a377b57efa034a174542a7d0abff64a6c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (xhigh)","source_effort":null,"source_model_version":"gpt-6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"652249b0db7e977c2330a7d79f1333bd4c79cbfe7d995aa6fa81c15153e577aa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.73611111111111,"score_display":"51.7","source_stated_rank":74},"run_fingerprint":"922a990ef349f53db7a4c2c5e6cee9939ebc7174622fec6291afc2a47d5fdf29","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"70e7f3038ad400c92c0fa017488e03bc652fe71cb49148b437b7391beca3f8f5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (high)","source_effort":null,"source_model_version":"gpt-6-luna_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"02a87a5eef5616ab25a0348270ab5c8af313606a26ae6d4876070e5e82205c0f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.3472222222222,"score_display":"50.3","source_stated_rank":86},"run_fingerprint":"870a918b8b9028627d2882ec8abbfa46284014ab7054fd832cfed3fb81eb6566","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0eee19e7d843aea97bf6f1966454da12e546f8043ee750c202677be2805a3e2b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (medium)","source_effort":null,"source_model_version":"gpt-6-luna_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0c02b76459341b3188d404bab51c7c0d6f4f3ea09f2be4c8d1a5edb8491317b3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.925925925925895,"score_display":"50.9","source_stated_rank":82},"run_fingerprint":"c565dbc1df68d480b9cb790a66703625e631982fcd43fc49775fe1b54a6e5cbd","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"12a80fd68d69a4f59b65adcc851b4a1f5c946b5df200f778bd41b2ee313fd7b4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (low)","source_effort":null,"source_model_version":"gpt-6-luna_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"fe8d5bf5e958239d83a33b560efe9c2a7ba49926e0d5a4f6c0254611697acb63","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.875,"score_display":"46.9","source_stated_rank":114},"run_fingerprint":"1978de78295fea9b738ead68d1627c9140a2ba9c7775315ba0fe4771548a5164","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f98412b891732ef67a8c1092aab738b5172a1ed269056470195f8f1053961be1","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna (Non-reasoning)","source_effort":null,"source_model_version":"gpt-6-luna_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"65b10f2d598fcde87fcf7a7259b0b6d99b730c2bdec2360fb683a3efd0f8f4d0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.0555555555556,"score_display":"43.1","source_stated_rank":132},"run_fingerprint":"74bb31080bf108bb606eca37eb20575263dfcb247d64ea671aef97d214b6a033","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ef9103e116e74d871e280bcad909ed2f83f687506b724538c6b2ae7939232cc8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Max) · scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"max","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna","upstream_model_label":"GPT-6 Luna (Max)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"983d61d355e50e911f52bbbb8b4d9b0fabc17d5745fd389e5b6dbdc55193ef0d","result":{"confidence_high":60.280942,"confidence_low":48.856435,"sample_count":288,"score":54.62963,"score_display":"54.6","source_stated_rank":null},"run_fingerprint":"ff7728a5bc005b721098f0296aa10683dabf2a5cae926dcdcfa5713a2e55b9dc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Max) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e2f095227912248274b9b8d70e2e00aea2e18b759ee0e964be68e4c8149ca6f9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-xhigh","upstream_model_label":"GPT-6 Luna (Xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"28aace81c082a969385ec70c2e2ff1905a4da540fe85d8809da0573ac7b5a214","result":{"confidence_high":57.446417,"confidence_low":45.980099,"sample_count":288,"score":51.736111,"score_display":"51.7","source_stated_rank":null},"run_fingerprint":"674fe24a742b5bec056ac28b24be4a24bb42e862a372eab860f12fd56aa06fd5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"64624a635ccc69bfc906c48fae96e580928e806e705d00ff97e046e3cb2a502d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (High) · scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"high","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-high","upstream_model_label":"GPT-6 Luna (High)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"304725efe56c2f57bdab76669caa7aee1e8ea50b127a9262dc61181b96807d5f","result":{"confidence_high":56.079087,"confidence_low":44.606216,"sample_count":288,"score":50.347222,"score_display":"50.3","source_stated_rank":null},"run_fingerprint":"353f848c47c90d3033da1af11c2ecd8de4633a95532f98ec932c176db0da4372","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (High) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1262db036724f375c4958ece1750508a38f2eb5c9cb97edb94ad5474286ea969","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Medium) · scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":10,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"medium","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-medium","upstream_model_label":"GPT-6 Luna (Medium)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e2d0c64f81eed759a9b17016c2f3f7168561a003e26240e6629a3cd072cb6b09","result":{"confidence_high":56.649339,"confidence_low":45.178136,"sample_count":288,"score":50.925926,"score_display":"50.9","source_stated_rank":null},"run_fingerprint":"6355da3c50e16d81de5126255264b71fb3adf26942eaa7c78177dbfcb0c08db6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Medium) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4b2629477c6d81603828f2dc6976c92635e42434a71e30205be92f30b9cf1422","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Low) · scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":11,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"low","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-low","upstream_model_label":"GPT-6 Luna (Low)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"d3f85de1d02a34c24aa4ac17dc6dbb221103851181543be8e1b2a78d377703ee","result":{"confidence_high":52.64164,"confidence_low":41.19063,"sample_count":288,"score":46.875,"score_display":"46.9","source_stated_rank":null},"run_fingerprint":"8548351d604e3b0c25a09d8f15aa5558cbc4265e1e542342ca24b775a24f3309","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Low) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ecb381772a495376804284a45d118b9df9190e1234bb6ca756dbe2a6aea575d2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","source_locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · scicode"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":12,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gpt-6-luna-non-reasoning","upstream_model_label":"GPT-6 Luna (Non-reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"571c33c053baec4ac4bfe43daaa21f8aa249fed427254fe8a5aaeaf62fb4255c","result":{"confidence_high":48.828677,"confidence_low":37.465259,"sample_count":288,"score":43.055556,"score_display":"43.1","source_stated_rank":null},"run_fingerprint":"7db0252b45c25060146c041c5332de2e6d1b217535ee68ec6e77d849d5cbd93b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:58Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:58Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"882200230b8574b52fa5d7cc4bb7995a146d5244f3e688774e7b806c55a95dbd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.018045","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-6-luna-xhigh","source_effort":null,"source_model_version":"gpt-6-luna_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"187f849bb54a75712c9a6838bd1bb2e80773d3ef45aa8518c75a7ed32b8d41bf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1576.92,"score_display":"1576.92","source_stated_rank":15},"run_fingerprint":"fb6d42e4b28f9417b1bf0dd8129fb701562493000fb21baded33824006067ca4","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fa544eaa3d733e08c61541a4104e4a91604acd8c7971c705eb88a6269c54e493","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.345703,"date_is_proxy":true,"latency_seconds":2219.515,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.377},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"max","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6-luna"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"c720050e21728e39e4027231ff1b5d5ad02c70c984910f8ddacf9ff04d6f93da","result":{"confidence_high":88.26792,"confidence_low":75.03008,"sample_count":null,"score":81.649,"score_display":"81.6","source_stated_rank":19},"run_fingerprint":"4973dd39a933cecf712eade2ee861a2cd85ecce5b91607ec82a2eeb233e5d784","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:57Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c61a7dc4b35c75521c95d1498a4db424b6747a50cefdd862e3bc5106cbcf27bc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-22","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-6-luna_max","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"47d0da06ea88cc6ba9f7b80185c9fde80e27bd845c79df05e95b2c205caca278","result":{"confidence_high":1590.96,"confidence_low":1572.94,"sample_count":5577,"score":1581.95,"score_display":"1581.95","source_stated_rank":24},"run_fingerprint":"50d88b68105e413334097ffe0accdbbaffc0033b9c8ebbbe5502405e8249233c","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e185a6ff7336bd2aee0f7ae8b0a59e86707255dd70887a362b96c7f0ec422511","metric_details":{"license":"Proprietary","variance":18.887958196145288},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-6-luna-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"977cbf8d941e61e9fa437dfc00fc6326be04a7d632c799254d981ec40ce896a4","result":{"confidence_high":1587.2314138847448,"confidence_low":1570.1952977055819,"sample_count":6429,"score":1578.7133557951634,"score_display":"1579","source_stated_rank":32},"run_fingerprint":"48f397417e4a78c4611f1b83c4975d074e1b512e965aad6ae73410e712e07310","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8d47069052bfb64d009b1fad0962852ce58b6ee8c6a7c7713ffee3eeee92e85c","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.029222,"date_is_proxy":true,"latency_seconds":425.094,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.716},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"max","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-6-luna"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"2192728ea1cbcb28c53310add493cb370ac7169189292b31a91656ffedad9204","result":{"confidence_high":76.39736,"confidence_low":69.67064,"sample_count":null,"score":73.034,"score_display":"73.0","source_stated_rank":23},"run_fingerprint":"87bba009354db77cf61e4fc344b883b73067f0680f2c3145d2d5f9f6181a65a2","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01T00:44:56Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T00:44:56Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"73e4358f8466c57b2be90043ba3306453a7574afe2ddc8fffd2dc394f4c1ce87","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.59","mean_standard_error":"4.9","model_release_date":"2026-09-22","notes":null,"standard_error_points":550.0,"upstream_source_url":null},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-6 Luna","source_effort":null,"source_model_version":"gpt-6-luna_max","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"ba4f535ad0cbfa9918ca7f561c960d11d788889b139f3a23fd57c415a487b02c","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":44.3,"score_display":"44.3","source_stated_rank":30},"run_fingerprint":"a625a3fe68c41cc1758bd981e9ef4e53f54d06affd392d9695a9f7181292e13a","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d3743ed9444420b4d6d5f18bbcc5e442b8f0d3515ffd8a940add719155c8d8a5","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":29241},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"GPT 6 Luna (Max)","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3d03cb25d69da59e4546af9170f8c4050dfd8293dc0a1e8a9ea1b5730f192206","result":{"confidence_high":0.024239269156937623,"confidence_low":0.002807534834065562,"sample_count":3443780,"score":0.013523401995501593,"score_display":"0.0135","source_stated_rank":24},"run_fingerprint":"145cb2d58a9d5367e5d65d6db63d21f6c090aa56a41ece072c514fac8bb41de7","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2bef12a13727db8ab7ae8f7584123535db03eb269be465ebe438d6622b407b28","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-6 Luna (Max) · gdpval"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"max","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-luna","upstream_model_label":"GPT-6 Luna (Max)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"2ae9b6f5597ada7ef571f89a5c7dbc5c6d8e7e9ab268b16d39d0426d3cb7b863","result":{"confidence_high":1461.24,"confidence_low":1415.53,"sample_count":220,"score":1438.38,"score_display":"1438","source_stated_rank":null},"run_fingerprint":"4f1bf5b1c484fe54516f269396f343e418d2ec758482c934a9e8285bcc1c36e8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Max) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_c5c378717d51c7077b3462a2","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"989f9301ea97be7b259d11125aacd88be4a8e34883776fa3775362c4c2ae47fb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · gdpval"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-luna-xhigh","upstream_model_label":"GPT-6 Luna (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"57ba73316c18c980789fd7974a852a276811aa91f90d3cec33be268f9cd88e1c","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1366.09,"score_display":"1366","source_stated_rank":null},"run_fingerprint":"476daa14c09fa62b123c47af608eac17364316ea30ffd7aab98b9044e38aecf9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c3fe93be8d4fa0fd59bc2d19","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7fcc59891fce95f4a71669f85d505932904aa66b4b0443c1e03fafe709f1e827","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-6 Luna (High) · gdpval"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"high","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-luna-high","upstream_model_label":"GPT-6 Luna (High)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"2f190a84572f47237041d62a2761edb4c811b7a676ca43d1d5e0c60b11f3ca2c","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1345.03,"score_display":"1345","source_stated_rank":null},"run_fingerprint":"7988d840d5d79e6657b112cd231d2955262c190109d01ef2ffc9f446845e7233","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (High) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d61809ed00b2b7bd9a4a86f","provider_config":{"source_label":"high"},"provider_mode_key":"High","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"86b1778b342033e0a9609ce2feb154f7aad1e1d84799349ad85196ed22316830","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-6 Luna (Medium) · gdpval"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"medium","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-luna-medium","upstream_model_label":"GPT-6 Luna (Medium)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"b7a18768d1a1729384c6e6392abeae523e77006dd5d877e28b0e37ec83339cc1","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1261.51,"score_display":"1262","source_stated_rank":null},"run_fingerprint":"6e3b684b33645b800ea431b5445cf56379a1b1d405c9807745d73c63c9ee6c47","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Medium) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_dcffeceb0fb6e91165fa5ae8","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a52b9d503dce4743564698caf7ea1e07fa3bd4531cb3080894fb90a9b54db901","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-22","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-6 Luna (Low) · gdpval"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"low","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-luna-low","upstream_model_label":"GPT-6 Luna (Low)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"90546dbf2f1170086c0d85919cd648c575bcdc805e4daa220938054e7d04dc6a","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1037.98,"score_display":"1038","source_stated_rank":null},"run_fingerprint":"d6c06fb64db039a38bb47bdc3e0287d16845cb9ed5e8f967f9eda867be7da009","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Low) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_e8cef4b393b683674a6784aa","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3546810d2e9acf38b4c622a10d561aa49859bbd8aa053e1e1b9a3a29e236772d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-09-22","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · gdpval"},"model":{"id":"openai/gpt-6-luna","name":"GPT 6 Luna","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-6-luna-non-reasoning","upstream_model_label":"GPT-6 Luna (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"894f29f3030af15454b03ec0e524880fadff15393e3d81ab808bab2e46a81926","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1051.02,"score_display":"1051","source_stated_rank":null},"run_fingerprint":"63095efc4122a4ce4327df99a12ccf193361939f4bc7c49dcbd548f86da84bb8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-6 Luna (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_664b3726ef23543d8f06cee5","provider_config":{"source_label":"off"},"provider_mode_key":"None","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"571bf106d272d36d20706899f0d6f0b644339f828473fbeb7a8d0d80ec1ac31d","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-03-17"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.4 Mini; model release: 2026-03-17","source_accessibility":"API access","source_model_name":"GPT-5.4 Mini","source_model_release_date":"2026-03-17","source_model_versions":["gpt-5.4-mini-2026-03-17_high","gpt-5.4-mini-2026-03-17_low","gpt-5.4-mini-2026-03-17_medium","gpt-5.4-mini-2026-03-17_none","gpt-5.4-mini-2026-03-17_unknown","gpt-5.4-mini-2026-03-17_xhigh"],"source_reasoning_efforts":["off","low","medium","high","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bc376e2d4a7e81c0d1246ec8d6e6c511ae1fdaf4ad47d67bf45c83619125f639","result":{"confidence_high":150.46,"confidence_low":147.06,"sample_count":null,"score":148.83,"score_display":"148.83","source_stated_rank":58},"run_fingerprint":"c795571ce3df847b5882dd233ab969149a802f5a8a922af4f5c904845a8b61a0","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0c4947d15e8e8b2e1d736e54","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"20dc142de24df7014dc5fee9b5d0d9ff5655a7ac3a7ba50714ec39f70a8644ee","metric_details":{"license":"Proprietary","variance":3.670545588793223},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1450.8231766557146,"confidence_low":1443.3131168080624,"sample_count":63290,"score":1447.0681467318886,"score_display":"1447","source_stated_rank":84},"run_fingerprint":"17f6c62ce5e0ba385f70b10cd89b8048854724161204523c5bcc97a2b0b69bf4","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"72eac3c10a1013f1bd580155070446e79367c946c0491948e7ea5fa7588d927b","metric_details":{"category_scores":{"Agentic Coding":41.666666666666664,"Coding":71.624,"Data Analysis":70.78566666666667,"IF":59.804,"Language":70.95166666666667,"Mathematics":78.462,"Reasoning":71.32375}},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.37396428571428,"score_display":"66.37","source_stated_rank":62},"run_fingerprint":"d1d801c1ae796707ce45db295c18168921b398beb6cff1ae0c54151d25f209f7","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:28:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:28:49.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c5778e655558d2363e47202a3436b3685c705508cf9e8d12e66d41e3df81d7ad","metric_details":{"best_score_across_scorers":"0.294","model_release_date":"2026-03-17","stderr":1.4414290540008265},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Z3pjKgAULY9j9wmzQpdqZG","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ3pjKgAULY9j9wmzQpdqZG.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Z3pjKgAULY9j9wmzQpdqZG.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"72fa00adc85d3388e9d9fc1c13acf750aeed030934a81665e85e3976b5089d87","result":{"confidence_high":32.22520094584162,"confidence_low":26.57479905415838,"sample_count":1000,"score":29.4,"score_display":"29.4","source_stated_rank":67},"run_fingerprint":"6b52230898ebd684adc0a9d9e3757ade7cafe106295972aa880a862f0241c4ff","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1699ddbba00814881e84854472b0de2b77a2d409d61a1341f5d7a820f7ab22b9","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.012743,"date_is_proxy":true,"latency_seconds":63.11,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.457},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":84.848,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"xhigh","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-mini-2026-03-17","zero_shot_accuracy":81.313},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"9037ddef41c8e0a563c2ca4a9605c1ef5b28563b1ca4482de96a14eb7d193c24","result":{"confidence_high":87.89572,"confidence_low":78.26428,"sample_count":396,"score":83.08,"score_display":"83.08","source_stated_rank":59},"run_fingerprint":"f7ecd00e765f5ec1b16f503423479d9093c3716ccbad4bb45530f3d5e738157c","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T01:00:11.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T01:00:11.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ca422f8905d1a8fffd4aa81a727338160a3b0d74026b6219f773e67802f4cb2c","metric_details":{"best_score_across_scorers":"0.8686868686868687","model_release_date":"2026-03-17","stderr":2.406315641682249},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gKB2rCD7mHzFN2MCt2YCni","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgKB2rCD7mHzFN2MCt2YCni.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/gKB2rCD7mHzFN2MCt2YCni.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"f55c0c9919a47fd5a56357c8b0fc21373499c68e2132e1451f41de3533ea986d","result":{"confidence_high":91.58506552638408,"confidence_low":82.15230821098967,"sample_count":null,"score":86.86868686868688,"score_display":"86.9","source_stated_rank":75},"run_fingerprint":"322edbccc097f49fad31a229e2a505b0f15a4e1cd0685c2fdc1a2185308fe35d","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":173,"at":"2026-04-15T14:03:19.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-15T14:03:19.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a5b863c5347cfe13d7fc5689fc0e152f130e34fb41098394fa9f92ca0bfd41ff","metric_details":{"best_score_across_scorers":"0.8358585858585859","model_release_date":"2026-03-17","stderr":2.2274469189069883},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"AQF46S9hBhosKPEMy3fyB9","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAQF46S9hBhosKPEMy3fyB9.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/AQF46S9hBhosKPEMy3fyB9.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"de721c760305323ea6006ac157b7f76c471411139ceb3b7e55957544304bfbc0","result":{"confidence_high":87.95165454691629,"confidence_low":79.22006262480089,"sample_count":null,"score":83.58585858585859,"score_display":"83.6","source_stated_rank":101},"run_fingerprint":"f7bfba7f42db04ffebae96dc45754c59ea041cb748d097623f091de44f9cf098","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:46:46.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:46:46.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6fd7c6180014e4b0f2b97528a04d579f95dc4f148f12faf2a8f81f8a63ed1e78","metric_details":{"best_score_across_scorers":"0.6414141414141414","model_release_date":"2026-03-17","stderr":3.416903640391528},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Qyz2Qt5x4P2XwB3w4h4jFR","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQyz2Qt5x4P2XwB3w4h4jFR.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Qyz2Qt5x4P2XwB3w4h4jFR.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"a617382b7bd4750dc3a1eda492b9f70a070859476212ef648e6f800cade9c1ea","result":{"confidence_high":70.83854527658154,"confidence_low":57.44428300624675,"sample_count":null,"score":64.14141414141415,"score_display":"64.1","source_stated_rank":201},"run_fingerprint":"4283820d0056aa489378766d327b87a1902c95a3e551404c15428eb4c00f03e5","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_b9d590a675150943bea9c5e8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"04cc0d46dd910cb64e76dfa2c67a77fdc145c44d9d7cb96517fe9a413de57ac2","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.00368,"date_is_proxy":true,"latency_seconds":20.001,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.359},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-mini-2026-03-17"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6533d0ab5c3ac9f1eb1287b72cf6c179c6d87fe127f2a35a3b6d62465b96a284","result":{"confidence_high":85.25764,"confidence_low":83.85036000000001,"sample_count":null,"score":84.554,"score_display":"84.6","source_stated_rank":63},"run_fingerprint":"34d9f5473b0ccdba00d0769f1f17bf9574c8633706b2d36b68a717bc6daffb05","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:56:59.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:56:59.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"240b369e8e7a1a80857b417becbca2efa6dba5a276ca393428e9ae99dd69f294","metric_details":{"best_score_across_scorers":"0.8888888888888888","model_release_date":"2026-03-17","stderr":4.737793696791343},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"QGMk6JNCYuUkFMF4zdVCsM","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQGMk6JNCYuUkFMF4zdVCsM.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/QGMk6JNCYuUkFMF4zdVCsM.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"46c32a6dbd5cc35e27c1a07fcce0749d6df3838e83710dce955b3c864b4e92ea","result":{"confidence_high":98.17496453459992,"confidence_low":79.60281324317785,"sample_count":45,"score":88.88888888888889,"score_display":"88.9","source_stated_rank":71},"run_fingerprint":"638804f464baa93f693d79d23ef55d9e41326d89395183e7a6f03af15e62797e","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":173,"at":"2026-04-15T14:17:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-15T14:17:38.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"7b52e641886ad392ae0c38e6c7848a4213e83188421c1e2f51cc17d2696b8d8d","metric_details":{"best_score_across_scorers":"0.8722222222222222","model_release_date":"2026-03-17","stderr":4.250833416666831},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Z5Sp2AKYzTBNdMEyTfTRuN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ5Sp2AKYzTBNdMEyTfTRuN.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Z5Sp2AKYzTBNdMEyTfTRuN.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"caad18979af7fd730507180e4ba59319ae1c61c86a639dbe6224fbdef6a50e1e","result":{"confidence_high":95.55385571888922,"confidence_low":78.89058872555523,"sample_count":45,"score":87.22222222222223,"score_display":"87.2","source_stated_rank":80},"run_fingerprint":"e02bd19bb0e2c6216afadb58f311ef91ef8a9edc8e28309180b81c336e8dfe91","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:42:28.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:42:28.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ea7999858308de84f4af85fda7e118e2dfa38a3bb82f7cbb7c808eb83ddd54d1","metric_details":{"best_score_across_scorers":"0.26666666666666666","model_release_date":"2026-03-17","stderr":6.666666666666668},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"R4NrJwtfdSQkGCWyKnrxgh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FR4NrJwtfdSQkGCWyKnrxgh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/R4NrJwtfdSQkGCWyKnrxgh.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"dabd97242b0244656206619224e63f531f44251719ac31d845cfac8a38533390","result":{"confidence_high":39.733333333333334,"confidence_low":13.6,"sample_count":45,"score":26.666666666666668,"score_display":"26.7","source_stated_rank":225},"run_fingerprint":"6e091676a322d19e8379c60428859ccfa87402ccf254c4c9cbb39282040431ed","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_b9d590a675150943bea9c5e8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-12T16:40:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-12T16:40:38.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"04437614ab4e71a0a4730d868c6bb2b3e5132c6fa7e936c53b8a84705f2020a4","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.512280701754386","leaderboard_private_problem_count":285,"model_release_date":"2026-03-17","public_example_count":10,"stderr":2.966059084324674},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"jjFRrQvsud3XsgYufr9f8f","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 mini (xhigh)","thinking":null,"upstream_model_id":"gpt-5.4-mini-2026-03-17_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"c29be5d216a0fb49320061c3ebdf535bfc5fd74d6084a85230e48503aa015a3b","result":{"confidence_high":57.041545980714965,"confidence_low":45.41459437016224,"sample_count":285,"score":51.2280701754386,"score_display":"51.2","source_stated_rank":47},"run_fingerprint":"8d1ffcd321074e9d36f2ca78d937473a73c2d3b5324769b4369ec5966fbd997c","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-28T12:28:53.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T12:28:53.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"550e96da059a0c749119cece078a16a626b8a2059d6a987256cc08785249ea7c","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.24561403508771928","leaderboard_private_problem_count":285,"model_release_date":"2026-03-17","public_example_count":10,"stderr":2.55425481026507},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["low"],"epoch_id":"8nsNpE6ETikLf4tSrNHmGs","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8nsNpE6ETikLf4tSrNHmGs.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8nsNpE6ETikLf4tSrNHmGs.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 mini (low)","thinking":null,"upstream_model_id":"gpt-5.4-mini-2026-03-17_low","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"fd1ad8a8c099a02eb62f29e58a99f249eeec414d10d0683fdde0306aace2af8a","result":{"confidence_high":29.567742936891463,"confidence_low":19.555064080652393,"sample_count":285,"score":24.561403508771928,"score_display":"24.6","source_stated_rank":77},"run_fingerprint":"75866591eec5b5d9ffd3a60b75c0b8356075cf4ce6acd2381f27c7c82f54f0e9","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2d199edeee6bd5f54c3804d0","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-28T12:29:04.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T12:29:04.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1156f7e77ccbb06982547d21a67d18fd98d6a0650a306312c0eba9e930a4035a","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.17192982456140352","leaderboard_private_problem_count":285,"model_release_date":"2026-03-17","public_example_count":10,"stderr":2.2389789548206736},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["none"],"epoch_id":"gDwrSBE5GvFXymdBYMnMyV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgDwrSBE5GvFXymdBYMnMyV.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/gDwrSBE5GvFXymdBYMnMyV.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 mini (none)","thinking":null,"upstream_model_id":"gpt-5.4-mini-2026-03-17_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"08500bcd2d4b74ec7891244c8221faa97318a494946dad53473a56d1ec7c479d","result":{"confidence_high":21.581381207588873,"confidence_low":12.804583704691833,"sample_count":285,"score":17.192982456140353,"score_display":"17.2","source_stated_rank":95},"run_fingerprint":"433df239cb148494a02660a2f1d9e825a2eba6748fda7e2deef83ecb542ea802","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_b9d590a675150943bea9c5e8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0d6e27a054bb718272b5b4db15deef397939c18b46de05a26f6dc97a5a811f32","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.75,"model_release_date":"2026-03-17T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4-mini-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-4-mini-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e80a40dce6d44e9548d8b5cf7f58dbd5b98a2218d63a513b4a980ca9d1acb61a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.9,"score_display":"19","source_stated_rank":129},"run_fingerprint":"41b03b7838f9f2f505f016065835366e7a846d90da8cc52ea2fe84f7ae4ee4f7","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"460cda2531520c746d3e3d3e9bb7de621a331501c88d3b856d239e6f7a863d73","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.56,"model_release_date":"2026-03-17T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4-mini-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-4-mini-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"80d80c4b6adad0b7494910225ee3201e31f5daeb8dd38f435b1de91baf160faf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.19,"score_display":"13","source_stated_rank":139},"run_fingerprint":"96946256098ac4b9da53eaeb3ec2379e07bc022cf3f7b88a18220d059646a754","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f9372a42f02b0bfdf5cc1dafaaac73252dfd9fc2ef812ebe0c71d07a4e92f867","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.33,"model_release_date":"2026-03-17T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4-mini-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-4-mini-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7c7062aa03b4d65c35c5840eebc2037fc17dd86ff3fef3c6a7526ccb6a98bc43","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4.44,"score_display":"4","source_stated_rank":171},"run_fingerprint":"f6c3c23025f113fe8a233de7f412bf1258cc1c7114e6408cb15823746f998c77","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_8cd328d93504026d558af6c5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aa06a984671bfb71744fd084522a1876264dd32d2029cf150dc4eb177ef4c63f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.06,"model_release_date":"2026-03-17T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4-mini-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-4-mini-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bd3ea968cc04a418e313eedc0723ffc6346d44bd20a9e5fa9a95bff16843480d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.11,"score_display":"1","source_stated_rank":222},"run_fingerprint":"b061c5ee796f8a927322aa38535077d2ffda1a0db75e42c46a906fd2d72c7a61","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2d199edeee6bd5f54c3804d0","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9328011096d99ee1fc6023285a3520ffff6a213b5d177a0f37d4cf4073511b96","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.75","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (xHigh)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"2932bf18d83484546ee518e147ef11640139ce3c0b7ba9020fbc33845798e701","result":{"confidence_high":23.264665,"confidence_low":15.19207,"sample_count":360,"score":18.9,"score_display":"18.9","source_stated_rank":125},"run_fingerprint":"6c015334fa6391e8b3b4fc8a9da5b18f355ae105d1f3123ae32d516e06d48e37","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6ae2f0607f07307acfd0522df9102eaac3e2f41f211e23ef3d7efabbf814a3c9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (XHigh)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f3968196a5154a64f85c375352d56b6bc1a9ecf345c8c2f34af05dabc35193d7","result":{"confidence_high":23.264665,"confidence_low":15.19207,"sample_count":360,"score":18.9,"score_display":"18.9","source_stated_rank":125},"run_fingerprint":"4e00859ffc8aa4c5bf5414b009a60899dcfb120272d461d1763ff1a2b589370b","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d6cddbfce5b2f115d50295e833b3a8a8309842f5af02cf7b79b90a83cd984a53","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.56","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (High)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"b8b3372702c8c9a0541f9df7fd394351c633261a415ae7653fe8f724e24d2535","result":{"confidence_high":17.07733,"confidence_low":10.079982,"sample_count":360,"score":13.19,"score_display":"13.2","source_stated_rank":135},"run_fingerprint":"1fed3880d9ab9cb3739f2d875a0d92b61a39e9c355ed61b0c98832a350c6e137","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"428ef8371c4712fbb8e56acde57df2af356b4ead0986ed96ae4bfcec515ae827","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.33","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (Medium)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"f9b627f0c21b907cc282acc9520c9677ee2dffacf67596084fd46d67e0c22774","result":{"confidence_high":7.091574,"confidence_low":2.750512,"sample_count":360,"score":4.44,"score_display":"4.4","source_stated_rank":167},"run_fingerprint":"c6a40a0a561f5df401bd6a9ce6233db6f90c7ac8f9e0cad74ef06529db099a1e","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_8cd328d93504026d558af6c5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f2fcdd96694a684f3a6ef6d5ed5d6adb9af07c806eb611d18d2936a24effcd0f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.06","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (Low)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"7f5e64dfe584b4ccb106bfa2ccf9783a4b4bfe485cecb4f2baa8de5500f375e9","result":{"confidence_high":2.820121,"confidence_low":0.432283,"sample_count":360,"score":1.11,"score_display":"1.1","source_stated_rank":214},"run_fingerprint":"37567c17b37f51aee5167b619b5e66be51d1f3773d47292071c50a51624d7302","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2d199edeee6bd5f54c3804d0","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-12T16:40:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-12T16:40:39.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3b9616f83a8d03232aa97a213f859e8e3b4f691a5d5aa122c2dceb3057bb1f18","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.0975609756097561","leaderboard_private_problem_count":41,"model_release_date":"2026-03-17","public_example_count":2,"stderr":4.691557088212523},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"jtQ6rDiNxjNKkVMEPBE3Ub","log_viewer":"","logs":"","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 mini (xhigh)","thinking":null,"upstream_model_id":"gpt-5.4-mini-2026-03-17_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"3a6f043e3447e4f181bc0555c74923aaa94035d15034c142b42db7f783a752ba","result":{"confidence_high":18.951549453872154,"confidence_low":0.560645668079065,"sample_count":41,"score":9.75609756097561,"score_display":"9.8","source_stated_rank":57},"run_fingerprint":"548ef69c33d8b943cb3e98c5884203fdc3934e06438f5916e3c151e72a9d7fd2","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9540122e70726f051ede741986c2a0f408a515e90e860a66150616bc3fbd1709","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.007061,"date_is_proxy":true,"latency_seconds":50.813,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.975},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-mini-2026-03-17"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8141a8094cdf24d65c80e1fd565ce9892b2e3ac9a454e945cac6185927f84df5","result":{"confidence_high":81.16,"confidence_low":77.338,"sample_count":null,"score":79.249,"score_display":"79.2","source_stated_rank":49},"run_fingerprint":"9df4cc97d0d2848bebfc1004efc5f6c3b31b00c278589a61be2906b45080d5cd","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0dfca88c0836be4a546be5eb0be3f251d4dfa9f33f6c8fb25e34764bfa0fd34d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 mini (xhigh)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"28dd821ba5b6445f23f6ed17bded25107b3b844485ca112657e924f83b2ccf5f","result":{"confidence_high":19.153356,"confidence_low":4.953024,"sample_count":71,"score":10.0,"score_display":"10.0","source_stated_rank":92},"run_fingerprint":"174b42dc3ebbfc73f8e2c10a3a20f53c5ee3c521d39b95cff87d0018f7280054","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T01:03:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T01:03:20.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"13c8badbebdf440a9fd0df05667fe53da4a9a02b0e156b409f994ed7571c2833","metric_details":{"best_score_across_scorers":"0.24","model_release_date":"2026-03-17","stderr":4.2923469599092785},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"oSkuSAA2Q6oWk6pKKiWnhQ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoSkuSAA2Q6oWk6pKKiWnhQ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/oSkuSAA2Q6oWk6pKKiWnhQ.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"4dd6b3cb148467cbedb0f1f619e5788a9733f5beb4e74b78a55442d28042e780","result":{"confidence_high":32.41300004142219,"confidence_low":15.586999958577815,"sample_count":100,"score":24.0,"score_display":"24.0","source_stated_rank":69},"run_fingerprint":"22aa50fabaa1f6803ce77b392720bd8646a5e3f554558259c0a032583360b1d5","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":172,"at":"2026-04-15T14:38:56.000Z","basis":"evaluation_started_at","evaluated_at":"2026-04-15T14:38:56.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ecff78a2966d0a6a512d01184ee8778a801bd1c8f6af7c4a80252dca7902b7f2","metric_details":{"best_score_across_scorers":"0.18","model_release_date":"2026-03-17","stderr":3.8612291966536914},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LsjWfG6McNoCdd6yE3vSjd","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLsjWfG6McNoCdd6yE3vSjd.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/LsjWfG6McNoCdd6yE3vSjd.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"2f0271c6be9a86e3acce9fd5829cba5e500df180b2ee8d08a67cf3e77cf40333","result":{"confidence_high":25.568009225441234,"confidence_low":10.431990774558766,"sample_count":100,"score":18.0,"score_display":"18.0","source_stated_rank":101},"run_fingerprint":"5a9d26f3a139983c487058e6be8979bc963d5f751f15633bdc3a27a3d51ac1c6","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:50:27.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:50:27.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e86b8bf19e11457081c001568a29a6aa090e3472980e08f23c8fe3ddaa7a520f","metric_details":{"best_score_across_scorers":"0.03","model_release_date":"2026-03-17","stderr":1.7144660799776528},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"E7qQRwqDdPtSFUcug9rPPS","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FE7qQRwqDdPtSFUcug9rPPS.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/E7qQRwqDdPtSFUcug9rPPS.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"692ce347f80a60cd3f2fcbc256d081950e94e66e85e68f84a0c4763bea1b1fc3","result":{"confidence_high":6.3603535167561995,"confidence_low":0.0,"sample_count":100,"score":3.0,"score_display":"3.0","source_stated_rank":180},"run_fingerprint":"49fc46062d1acfe614f93ceb91760970649eedd1fdc74008ecc9676e0bf8f623","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_b9d590a675150943bea9c5e8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1e6a34ae8a3ac048fcb90287ad4d721e7622cd56b0f3076272eefbcdc4019a4e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.51","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (xHigh)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ae6f6765b96f58fc9143382fe604f5c2fa58622e65ef712f44f33e10871bf936","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.67,"score_display":"63.7","source_stated_rank":143},"run_fingerprint":"05ba46dc1bb0457667469b573cba9bd3e6b51e6abe88d1b214263b169677ceb1","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"88952508ed32be597c41e1834c0a7df39f8baf7a3245b6ff4b086156d9a63395","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (XHigh)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"7fb916034e51da5dd847d150eaf3d857907102da448ae49b42e44f610be1a460","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.67,"score_display":"63.7","source_stated_rank":143},"run_fingerprint":"e73f053af278dd87514a084001896ec18bbdfd20bb7cb544ca04eec33affa4b4","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"032d2ee2c0af74714aed9be55a51cf7322a15648392815502a941f3cc2d2e4ae","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.27","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (High)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"dbb63185e9b3ad28c7f974df7e57a4e9f8e833efb58548f7640b548c7e1027a7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.99999999999999,"score_display":"58.0","source_stated_rank":153},"run_fingerprint":"e5688d6b44353a94d495803c271af814fde1ee2f383801985523d714b561d5bd","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2bec8ec1fc15a06a171f7176d1119abd5d9a9b6ea93a833939ced56cd7a7a696","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.16","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (Medium)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6fea84ff0a905832d6ce7550beb3cdbdc21df0b71b6b2d4e3fc44ec8824d98b1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.83,"score_display":"40.8","source_stated_rank":180},"run_fingerprint":"8bd0c3bf62ad2fce72ffbf7c79fe43f4cc9426f7bcfc2e0c520d0543e8459505","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_8cd328d93504026d558af6c5","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"03cf00fcfb60960999a94f446f263d3e71383c5e2b44052f91e3445c76295eee","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.04","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Mini (Low)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a314fcfe355e17d91cbf646f855928fb5b1a8625efbdd78a75b2af766aebf556","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.0,"score_display":"13.0","source_stated_rank":243},"run_fingerprint":"be1c5724884c05b7b713624608cd53a83ed63840d73c5a2a670e0560f80ffdea","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_2d199edeee6bd5f54c3804d0","provider_config":{"source_label":"low"},"provider_mode_key":"Low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"849f2e3f758cedcf581c70d8a41a91b4cea167e3ea8662f3fe432efb06106fc2","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.511052,"date_is_proxy":true,"latency_seconds":326.46,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.987},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-mini-2026-03-17"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"feff89e5a1627a5c66e74d182acdd8555f76c77c9993f7e96c36a55eda8feab8","result":{"confidence_high":76.89452,"confidence_low":69.10548,"sample_count":500,"score":73.0,"score_display":"73.0","source_stated_rank":53},"run_fingerprint":"b8cd3e411f461c576ac3c3fbd0cc52257c4c137fdbc8dfd5077feb7c72885073","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"140fe81cc701041dff7ca0b63232678be02ef6ff615c3cdbe79afafbf97f32ec","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4-mini","source_effort":"xhigh","source_model_version":"gpt-5.4-mini-2026-03-17_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"codex","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"a914ef90457ec38e014bd6d0c4fa13247b98e205edefeaef436af0da0b0713f1","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":27.04,"score_display":"27.0","source_stated_rank":29},"run_fingerprint":"d99262b29c85a9790949fb6f0114d668b46edac3f7fdfb9d4b9705ca3c76ecdf","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"550d97263b5e8df056644b9bbe8f2b3a581b821cfc9dbe48599a95fd9854cdb3","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.032935,"date_is_proxy":true,"latency_seconds":188.757,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.094},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-mini-2026-03-17"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"3f386c9042f0b2a6bf0e9102097bf81e35fa983a598ec9cafe9cc1af52be8926","result":{"confidence_high":83.60924,"confidence_low":79.32076,"sample_count":null,"score":81.465,"score_display":"81.5","source_stated_rank":62},"run_fingerprint":"eb662f48e8e1164752c84863afec329d7690ec33bf932bc60fa56efc191b9078","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b3c7849f0e4f89aa801739448f80595767b6200078b6e7c87c5b4aea0d1cef37","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 mini (xhigh)","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"d47bbdc15f321640239043e66ef30edb8ebad1248732d7b05c2b4ac18a380083","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.88425925925931,"score_display":"49.9","source_stated_rank":94},"run_fingerprint":"02fb9cfd80218f14c008d4ae077dc2c8192f0677faa16c21f80117774bacdc5d","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"39e8163ded3e4782b5fd7c1f97e60d75cddbfc7417b4588ea3c143376737d961","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.271408","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.4-mini-high","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"5d3e6c572090ddf18d17752ee4d05457b8a476757b729a0a74b381f830b00dcc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1188.58,"score_display":"1188.58","source_stated_rank":37},"run_fingerprint":"512d3501b75a0f0140c6b2c85b2d532e83c9a6afeebc901281d6c38c16532fe3","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"21939a73c9232a66d82eec133509224c3675bd5975b4cfaa65c5312064e58a66","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.191299,"date_is_proxy":true,"latency_seconds":2055.117,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.606},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"xhigh","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-mini-2026-03-17"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"7cb67b8e5a43ae6fcfe0ef1aeed74d8a551c257a282c24ca7a067cb3859d5c40","result":{"confidence_high":58.95676,"confidence_low":36.98124,"sample_count":null,"score":47.969,"score_display":"48.0","source_stated_rank":55},"run_fingerprint":"3e66e27b29f433fb76b1a6fe5ed5bde81ab1abc168543d022238ff54e71778f6","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"522f1c4c86e00f6bde8bdc55c56e1b3651bf43e8c8b305c363816f25d0bd8bd1","metric_details":{"license":"Proprietary","variance":11.033867844723403},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gpt-5.4-mini-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"09d542cae296a4afb0a7d4e73e5b95274838d74ec6738aa2839d6f8b1c8cebcf","result":{"confidence_high":1403.5270237467612,"confidence_low":1390.5060946004235,"sample_count":11998,"score":1397.0165591735922,"score_display":"1397","source_stated_rank":87},"run_fingerprint":"0bcb4020cc83abbc19139734007c4d287f1d8c72bf395fd18f55bebcec0728af","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cfc41dfe503e0d8d86507b81cbc8e2366eeeabb4db2557ef042fe2b2f56947e0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_high","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"133304f1885baeba9562abe3758a67f25776644d0907482d17a9e883fdc35079","result":{"confidence_high":1403.87,"confidence_low":1390.5,"sample_count":11279,"score":1397.18,"score_display":"1397.18","source_stated_rank":85},"run_fingerprint":"22201204790194834bb5631896ff595cc5c914e21bb358bcd42b66b69b249ca7","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"616a47c69ab8c71f0c6c59f958f53d3d1cf89a0119d20fb4140bf52e7e9def1a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"08670d47312fe006c5a06588bea2dbeccbfd7b97875e82e7944fc1ed29ca942a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.3,"score_display":"60.3","source_stated_rank":51},"run_fingerprint":"c106952351c83d3a4e1b7e52df07d3288201dad60969b3c8fa17e42511f69389","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a4fd650e4f93946aec635cf869e2d0b7abc2aac9674af7bfc09ba5dc03168a8f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"0a9d5f409eb768befee9c77e21b1e253b3510a3db9ff5b3cc4f9a90de500812a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.9,"score_display":"37.9","source_stated_rank":127},"run_fingerprint":"a2ee2aeda72164c080d37e50c5ea9697a4ca77f384da9f3d9aa7c10f054c6e2a","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_b9d590a675150943bea9c5e8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"abc79fc59bfdacebb7e96ac0743db790657c6c92d77fcd7c0d23d684db421402","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.286739,"date_is_proxy":true,"latency_seconds":673.036,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.556},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-mini-2026-03-17"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"17fe14f9447ea0f6b3d6d05bcbc346996cf0541bd68d9e4400e20fad75d2957a","result":{"confidence_high":63.611760000000004,"confidence_low":45.75224,"sample_count":null,"score":54.682,"score_display":"54.7","source_stated_rank":51},"run_fingerprint":"538e01947377193a3d1985b6725cb9cfb6c0a8bc43779c6497547e1f4b6c4cdf","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cd463b62537d29712133caab","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"5e27ceca0d7c694bf196c1c987c4d9833e418600e361a562e5b5f95c55bff6fe","metric_details":{"expected_trials_per_mode":261,"normalized_gain":16.4,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":11.5},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":49.599999999999994,"confidence_low":33.2,"sample_count":87,"score":41.4,"score_display":"41.4","source_stated_rank":18},"run_fingerprint":"0befa8ccd464ba5d0240bc986890964300ffdddbb51089f3b7886c9c61a9b0cd","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0ad87315d87f37ec5bf6e958","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"50a805c445c84dab116b30c75c88c60640ba83c83d75757e71cb830fa3cff56e","metric_details":{"expected_trials_per_mode":261,"normalized_gain":16.4,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":11.5},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":37.3,"confidence_low":22.5,"sample_count":87,"score":29.9,"score_display":"29.9","source_stated_rank":13},"run_fingerprint":"dc04806e56959b4b82ec9f722b912d9ae6d7bde46dbd284cb6f98c2616ab1197","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0ad87315d87f37ec5bf6e958","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"d233fe7fff51d765bb37efed7231c1b9656b647e475a3377f416e0673a23cf1a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.375","mean_standard_error":"3.1","model_release_date":"2026-03-17","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":3.2,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5.4 mini","source_effort":null,"source_model_version":"gpt-5.4-mini-2026-03-17_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"de045ffbfe7418ca98c684271624c6fce0b8aa52f1b32576fe23c72ffc1d0e47","result":{"confidence_high":30.872,"confidence_low":18.328000000000003,"sample_count":480,"score":24.6,"score_display":"24.6","source_stated_rank":30},"run_fingerprint":"c48a2ac3d673d1856d12d9fca502f4db3b952ac4138cbfe71e75318abddaee0a","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6fe262c772c78fcd1006e216","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"bcd0d9ac9b82d11e54778a02d7d193a97c9a9d1221b4d83360d7ad80202b61d9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":400000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-03-17","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · GPT-5.4 mini (Xhigh) · gdpval"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-4-mini","upstream_model_label":"GPT-5.4 mini (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"47086e3e999943e108c80edf48f94b61da9203be81722bf37a1e58dd98dae842","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1000.22,"score_display":"1000","source_stated_rank":null},"run_fingerprint":"9351c1f5634578e79c307021eb7131febcf53f422e463438a5739721362897e5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.4 mini (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_c6f0f8947c7f8dc16f5517b6","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-01T22:31:28Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:28Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"abeafd74fafc56aa5a1198d7deb1c73345f13bbbcda7f8b0317e84f1e4d30372","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":400000,"is_open_weights":false,"is_reasoning":false,"model_release_date":"2026-03-17","reported_confidence_interval":"-21 / +21","source_locator":"Leaderboard models table · GPT-5.4 mini (Non-reasoning) · gdpval"},"model":{"id":"openai/gpt-5-4-mini","name":"GPT 5.4 Mini","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-4-mini-non-reasoning","upstream_model_label":"GPT-5.4 mini (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"a133ab5e831270afd00ab109d8f5418b9b576c6811a9c8f4326f82ce0014bf54","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":595.44,"score_display":"595","source_stated_rank":null},"run_fingerprint":"2395109fdec4c83fecab5789419d3dc909a3e67f546cb5779577a473d7329e7b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.4 mini (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_b9d590a675150943bea9c5e8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"53098d312d1daa5914bc68b2a5313d8d9a4455c3ca5cda182d692b7581470beb","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-15"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Inkling; model release: 2026-07-15","source_accessibility":"Open weights (unrestricted)","source_model_name":"Inkling","source_model_release_date":"2026-07-15","source_model_versions":["Inkling","Inkling_high","Inkling_xhigh"],"source_reasoning_efforts":["high","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"43df354df0c6c71c6f49b892b50c335e8ade74f0a079615a362239ece6c8c14f","result":{"confidence_high":150.63,"confidence_low":145.68,"sample_count":null,"score":148.59,"score_display":"148.59","source_stated_rank":59},"run_fingerprint":"c30e7e0c98400e4b7f7704336da4aa124d4e5db3ff5f59aaf2b6faf7a8a1cd2f","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_57fc6d9a28dcfc60bef75504","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aa635bebaa7e71301fb680b1f8452c3d40bbd6eaabfa4f78b916f5575ba75529","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-15","source_locator":"Leaderboard models table · Inkling (Xhigh) · intelligenceIndex"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling","upstream_model_label":"Inkling (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"1f46abe4de429aaa685f0b89e5a51b708f3559220ae50b85d72fbba828e97f87","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.984781,"score_display":"25.0","source_stated_rank":null},"run_fingerprint":"84c5c15d2dd48020ad33f9d33f4063f685acd750f51ef64f0a9b57d706220da0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling (Xhigh) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fbaa0f6c8baaaade10c1e5c3c3ffc658e2e068c33d330b32d54a9de012cd3e13","metric_details":{"license":"Apache 2.0","variance":5.278507311035362},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1445.8784979977504,"confidence_low":1436.872462583635,"sample_count":35204,"score":1441.3754802906928,"score_display":"1441","source_stated_rank":95},"run_fingerprint":"1110e876ebf68a9ac016d67ad7e32d098cbef53d37a8ab6e8367886d4910ebcd","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b73fb4763ea0cafd73adca99","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5fd71c1cc5edca1238b39d988f416cb086a91d49b1b223fef4ac74cc3f47d959","metric_details":{"category_scores":{"Agentic Coding":49.394000000000005,"Coding":71.0195,"Data Analysis":72.778,"IF":70.09575000000001,"Language":73.45866666666667,"Mathematics":88.36474999999999,"Reasoning":78.34625}},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":71.92241666666668,"score_display":"71.92","source_stated_rank":51},"run_fingerprint":"cbff60eceba60eabcf11249e9b5c0b201d3434927e45c96ded98ddf61571b705","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aac46513cea3968221c26e1eace0daa2563c5c6ede6e830514cae28b5b8868a9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":null,"num_parameters":952377623626,"pull_request":null,"result_file_url":"https://huggingface.co/thinkingmachines/Inkling/resolve/main/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/thinkingmachines/Inkling"}},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.0,"score_display":"46","source_stated_rank":21},"run_fingerprint":"39dbf2cd95dde4bbb09573ffb168dafbd524c302bac054c2f2b3c888f7d9b717","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9917693906cdf9a9f6c99327","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":82,"at":"2026-07-15","basis":"evaluation_at","evaluated_at":"2026-07-15","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1585786f1bae7ceaf23b93a966f60b1b443190c235a713ab0a896ca4e40272db","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/thinking-machines-inkling","source_model":"Inkling"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a7a0180c9f9eefb8783a356cbf9eccc66aa8407e1c8a4727d6a4701eef613712","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.7,"score_display":"29.7","source_stated_rank":28},"run_fingerprint":"5fdcf6a5549fcdf8f06b30c034cc80c9dd13152698179aa4c061180fa12503db","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d01690983c536a01840aaed6","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6422153c204cae56a6e0af943a56410e2e0c6d1371a526173875a930cc538c88","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-15","source_locator":"Leaderboard models table · Inkling (Xhigh) · hle"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling","upstream_model_label":"Inkling (Xhigh)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2cad4b8583d9ea9085c09a8207260600a887349be2f24eef053c455e33a353ef","result":{"confidence_high":33.878304,"confidence_low":29.948834,"sample_count":2158,"score":31.881372,"score_display":"31.9","source_stated_rank":null},"run_fingerprint":"7a1b1cb3a0e3bc87b0f3109375a931e9fd400742e427b4110a0cd7e80c3cd859","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling (Xhigh) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:36:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:36:45.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2c184c7f2ec325d582fb22a4aa0faa67f977eb54b3c4b7f290cd7f690772ccd0","metric_details":{"best_score_across_scorers":"0.403","model_release_date":"2026-07-15","stderr":1.5518757419066513},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bpnzC6CCi3hWuKCfqX4KEY","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbpnzC6CCi3hWuKCfqX4KEY.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bpnzC6CCi3hWuKCfqX4KEY.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"20eb758f84971175d825a75c364c42137740841325eaf7993f5a8ad3ae1df78e","result":{"confidence_high":43.341676454137044,"confidence_low":37.258323545862964,"sample_count":1000,"score":40.300000000000004,"score_display":"40.3","source_stated_rank":47},"run_fingerprint":"9a0ce76cbcae5ac061ceaa41612a8a4485a509709bdce29bc0d28f8ce219a976","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"384b0a6c717cf8b5e910aac88b503f97bce193d61bba79231904296dc458c86f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.16589,"date_is_proxy":true,"latency_seconds":308.595,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.784},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":87.879,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":256000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"0.99","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling","zero_shot_accuracy":86.364},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"7ad049b59ef2f5c376a6b2f0a0a292801a49d6b6344cbce2e4f7487d7d8b7b9a","result":{"confidence_high":90.61864,"confidence_low":83.62536,"sample_count":396,"score":87.122,"score_display":"87.12","source_stated_rank":40},"run_fingerprint":"ec559b5b69793214e6a85f1628982ffb916f901ab13907712eb545d7ee4e154a","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7279a3f7a28bd9930d9b03cc","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8742b1663c30beb77f19f57e20a3f1baf4371c3cf17605abc83415c66dfa92a9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-15","source_locator":"Leaderboard models table · Inkling (Xhigh) · gpqa"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling","upstream_model_label":"Inkling (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5794a18338dd073d3367d26558ea5f3e834d17edeae767a4b86b411c04053cd6","result":{"confidence_high":91.131587,"confidence_low":81.796888,"sample_count":198,"score":87.171717,"score_display":"87.2","source_stated_rank":null},"run_fingerprint":"d9aae071fbda3489d063839a48ac25e60e751695c0d9d27d47341272d4c47aba","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling (Xhigh) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":60,"at":"2026-08-05T18:55:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-05T18:55:39.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8a8591d762ca60e45cb6fcda9cb4d45172cb61696c0dedd7ff6d7a0a2c5c0d0d","metric_details":{"best_score_across_scorers":"0.8825757575757576","model_release_date":"2026-07-15","stderr":1.9355903133033585},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dfmiZmjAcNkLuAJL4z65eb","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"3a195fdc46a1981405ffa131ddafb2fd9ab33bc3dcad35f46708103deb2329db","result":{"confidence_high":92.05133277165034,"confidence_low":84.46381874350116,"sample_count":null,"score":88.25757575757575,"score_display":"88.3","source_stated_rank":61},"run_fingerprint":"255bb2ecbaf0b6e6f94236ff86d71ac6162694a67333e549b871d6e49eddc2bf","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cff69062147b0c53d44f2a5f0284c09547c15ed8ec3f7747912606b655329f0e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.039629,"date_is_proxy":true,"latency_seconds":73.264,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.34},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":256000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b213888c0252660b638c19d000e70c2e8b7f0b8f6904b604118bbdb7ed64119d","result":{"confidence_high":86.9614,"confidence_low":85.6286,"sample_count":null,"score":86.295,"score_display":"86.3","source_stated_rank":43},"run_fingerprint":"233f9ee196dda42bab5a3765a58625c55bc980466458951ebe7f14b5f73a2a4d","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c8d9902582e198f49805a576","provider_config":{"source_id":"vals-mmlu-pro","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":60,"at":"2026-08-05T18:55:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-05T18:55:39.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6fda395337986f6e06468db3fc7e460c6fdd0040001d8851113cfe0ca41822c8","metric_details":{"best_score_across_scorers":"0.8888888888888888","model_release_date":"2026-07-15","stderr":3.9171949640122654},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"j2fELMf364qfh2cpaVDL6W","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d7ccddb87a309ce5d6c745f39bb12910ef2327bf966e1c993a0a4c524bce446f","result":{"confidence_high":96.56659101835292,"confidence_low":81.21118675942485,"sample_count":45,"score":88.88888888888889,"score_display":"88.9","source_stated_rank":71},"run_fingerprint":"5638ccdd36ebea91fa21b0744b1ddcc2b558cefa8d5d2e4edccd7d09af7f9f4b","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T20:14:32.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T20:14:32.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ffdf3a1624be778fada717505bb2824e3131ba9771c6343108d357aa2a4e8c53","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.3333333333333333","leaderboard_private_problem_count":285,"model_release_date":"2026-07-15","public_example_count":10,"stderr":2.7972711943222968},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"6KvQvuMMX5zAuteUc6pFcW","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6KvQvuMMX5zAuteUc6pFcW.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/6KvQvuMMX5zAuteUc6pFcW.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Inkling (xhigh)","thinking":null,"upstream_model_id":"Inkling_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"71b356cdd9bc989e02c5e1c7ada49f7370ae5ef3e39de146b938b0bd88eb988f","result":{"confidence_high":38.81598487420503,"confidence_low":27.85068179246163,"sample_count":285,"score":33.33333333333333,"score_display":"33.3","source_stated_rank":66},"run_fingerprint":"e6096a820b35b5a89930fdaee30c311785efd9b0cb616b728b77ce3677427872","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2d1ec8184885ec26bd225505b5f1843b729ccf26feb44a0d80e453cd0f846dbe","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.6420766277310923,"model_release_date":"2026-07-15T00:00:00.000Z","results_url":"https://arcprize.org/results/thinky-inkling"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"","model_type":"CoT","upstream_model_id":"thinky-inkling"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"84ab3445a8f1e639d5912777a39d57baa63d5da37edee2657435ca577434367b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":36.52777777777778,"score_display":"37","source_stated_rank":109},"run_fingerprint":"42133160f0544940f30a6558f1ba80595b952e74d1d9c73b589c661be7687802","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9f4576349b0b9a1dcba8e325","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b7d4c9432980b5b24ead08919bd360331298d27eafe80de700680bf7254fac8b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.6420766277310923","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling","source_effort":null,"source_model_version":"Inkling","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"17fc1e50e1bd8e81d2b245a85c632f108aa1b1114ed54a2b2f0f92c471189c0e","result":{"confidence_high":41.619768,"confidence_low":31.720279,"sample_count":360,"score":36.52777777777778,"score_display":"36.5","source_stated_rank":110},"run_fingerprint":"9933e0e76e7c867872becde74f2c4e6b52d7e59c15d3f2a8c96d85affc508fec","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f09fc9effaad7eba0d379d1a","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T20:14:32.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T20:14:32.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1b26c0eb1c2b0e44cfffd058ca8588d86abd05d2a701b4033967444040559171","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.04878048780487805","leaderboard_private_problem_count":41,"model_release_date":"2026-07-15","public_example_count":2,"stderr":3.4059122057973035},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["xhigh"],"epoch_id":"jvJPoRYkPH9Sb5MX252oHp","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjvJPoRYkPH9Sb5MX252oHp.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/jvJPoRYkPH9Sb5MX252oHp.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Inkling (xhigh)","thinking":null,"upstream_model_id":"Inkling_xhigh","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"5d06392ee872e180a995f19d965db004f09484e9081d09704fbcf92473a6e0a0","result":{"confidence_high":11.553636703850518,"confidence_low":0.0,"sample_count":41,"score":4.878048780487805,"score_display":"4.9","source_stated_rank":58},"run_fingerprint":"6014c27e710c0ce5144e866c5eec8babca9cf9d40e8eb374399099035578c75c","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e92e6d20438cfb0a8d6cb8aa08f74a2cd4f8d2e4c889035ce045d59ac8fc666a","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.031466,"date_is_proxy":true,"latency_seconds":123.645,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.988},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":256000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e0c79a0f5d6d52fd970dd3e752a0f67a5f779dc87794d5abf7ec11cb14c72476","result":{"confidence_high":80.43348,"confidence_low":76.56052,"sample_count":null,"score":78.497,"score_display":"78.5","source_stated_rank":51},"run_fingerprint":"c5eba924ae234c795c04cdb9a008dd614c957637d85cec1ca847fc32d78f7999","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fa698e6a574cf4d42e206b07","provider_config":{"source_id":"vals-mmmu-pro","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"24072fb024659d0e410da462444c3f5f6c1fa7e0faac252f6b1da3020f18527a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-15","source_locator":"Leaderboard models table · Inkling (Xhigh) · mmmuPro"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling","upstream_model_label":"Inkling (Xhigh)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"bdf78148adb6c0c9856fe661a962f387bcd4d61148f797c1f90158ad065cec5d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.468208,"score_display":"73.5","source_stated_rank":null},"run_fingerprint":"cb61026c5bf78d82f78d0195cb428392cdc8d8a0984f228f7f2d803688f3565a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling (Xhigh) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"79d2b301031c85450316698ac07d656959d841bba585e0be7f408c9cf50f27ac","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling (xhigh)","source_effort":null,"source_model_version":"Inkling_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"388810991199af905dad38521ae4defd58d96491cc65c54d3583a4890b68c20f","result":{"confidence_high":13.336578,"confidence_low":2.096245,"sample_count":71,"score":5.42857142857143,"score_display":"5.4","source_stated_rank":108},"run_fingerprint":"a7d0730458fddab58419ee058aa263bb0ed2f3de02183dd9e0924b41723a0fd8","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"007d0bb6dae1b91e6eeaba929b65f5377ae495b9191484e2c1d203554d5aea4f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-15","source_locator":"Leaderboard models table · Inkling (Xhigh) · critpt"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling","upstream_model_label":"Inkling (Xhigh)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"0229bca0909c11cb26eeb3880a21f91caf0a77d867da22d5fb8d1e20d442bb4d","result":{"confidence_high":13.411833,"confidence_low":2.082955,"sample_count":70,"score":5.428571,"score_display":"5.4","source_stated_rank":null},"run_fingerprint":"ecb736395add2701e70b5163e54d45808656c3eee33eebd2868c71009109d765","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling (Xhigh) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f27d074a9acd554565bc36d0689bf275d8291da1b2c07427e7ec9030fafa37a2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://lmcouncil.ai/benchmarks"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"Inkling","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"f6b61380000b7f489f75162e5da21a7d50a80ee616f5a65ccd9f104704b53447","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.0,"score_display":"50.0","source_stated_rank":51},"run_fingerprint":"523fa569eca755f3fb0098719df410c6903424229f5df40f168486fff7a8dae7","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_323430f8d70d985199accebe","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":60,"at":"2026-08-05T18:55:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-05T18:55:39.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3a272368a16c020a8ec70dfa7a54851c4815b3bd1971d713ada9cc8a24e9f79f","metric_details":{"best_score_across_scorers":"0.21","model_release_date":"2026-07-15","stderr":4.093601807403323},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"VYD3GMYKSkMr75q4CMLDDq","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e2b899c2fd97b946392ef72eed3ba65a7793fd506d783f58be069a8fafa06c44","result":{"confidence_high":29.023459542510516,"confidence_low":12.976540457489486,"sample_count":100,"score":21.0,"score_display":"21.0","source_stated_rank":80},"run_fingerprint":"1ab054244e8b02580dce81cfea9fc22eeadea929a55bc6c534c3596f2b019158","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"628af3f7409917bd2158c882625c4e1b6bf18c3454b7ad57c6e76beedbb5ac91","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.30047416199999993","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling","source_effort":null,"source_model_version":"Inkling","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"293efacf877896e53be0b51a4aef786c41496191bf99b1897934d470ed42dd8a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.5,"score_display":"79.5","source_stated_rank":113},"run_fingerprint":"06630ef7374fa40c182a6af7f67ab45c8ae68130431b895f58209cc5bc68422f","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_84441dcf4ff9d3a7ebb4f5f3","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"aa2001cbe2f418f7b951a406f0a2d7604215a786d26d68a9db706e0b8779a1f9","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.585966,"date_is_proxy":true,"latency_seconds":394.193,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.866},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":256000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"2e66b1ecf93d370908ab69f72d08b4d02d16ef34f759a5dd4c221bf8afbe0151","result":{"confidence_high":81.25735999999999,"confidence_low":73.94264,"sample_count":500,"score":77.6,"score_display":"77.6","source_stated_rank":34},"run_fingerprint":"9ad27042465d94f8bb0106a4df2b39cd3c133e9b559507856964c2fa0edb9c0a","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20afbe6979e993a50873f13e","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6822c0c7aac2d04c567d1b2a8b26345a4d733b555648041c64d2a6177c5e792c","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling","source_effort":"0.99","source_model_version":"Inkling","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"4f671b8d19afc203b8b3147d4f5f2816d40402fbcd2c142155ec201ac578736f","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":14.000000000000002,"score_display":"14.0","source_stated_rank":37},"run_fingerprint":"15c00966c823b47951cc8409dbe5a4496699f55c2f72f9832517f3bfa31182f2","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b8c36c61e899913b2bb3beed","provider_config":{"source_id":"epoch-frontiercode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5e94be50f589f8790a84c0b1bf51c243b8d2aaf989864c6dd3731c32860afd1d","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.70838,"date_is_proxy":true,"latency_seconds":1812.242,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.403},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":256000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"ed5c72f9c0d78dd16b9879d3460ac11000e7ef2ed1d83c5ddac0dc9f71b4c9b0","result":{"confidence_high":25.53388,"confidence_low":4.354120000000002,"sample_count":null,"score":14.944,"score_display":"14.9","source_stated_rank":38},"run_fingerprint":"89e110a10075d88c6ef70b3028dad378ae354c48a85fc53d7a00e3f058cd50c0","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_aa19ff77759412701a03b4d0","provider_config":{"source_id":"vals-ioi","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"043d3119ef7178a371b659f505401bc416398ef5453245d911aabdfee60b92b5","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.153251,"date_is_proxy":true,"latency_seconds":257.712,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.003},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":256000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"fff72ae1d325a096463be1f1852b0d6188e93cefce8be7bd11df8905d7437572","result":{"confidence_high":87.48288,"confidence_low":83.55112,"sample_count":null,"score":85.517,"score_display":"85.5","source_stated_rank":32},"run_fingerprint":"8e146dee33b8f5f601b53f23e4fd85b3fc08fb2c1d10db40cc8652fd0a902399","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d9ea177d067409c4e24792fd","provider_config":{"source_id":"vals-livecodebench","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6ed8894948c87589405291c12b34bd931d601c2ce9eca9928c552578259c61ee","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling (xhigh)","source_effort":null,"source_model_version":"Inkling_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"221e615e36cfc53eb5907902169b250265600c6c3e24614a04a85050e6b30d5b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.9907407407407,"score_display":"47.0","source_stated_rank":112},"run_fingerprint":"4263f87f447b73b11f459aeb81d06ebdaaf6e1ff8c72c8742f2bdd2b820690f5","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eb88908a20dcbcef0faa064a20b2ceb9ed41c744e0c42c77315de53178b96eed","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-15","source_locator":"Leaderboard models table · Inkling (Xhigh) · scicode"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling","upstream_model_label":"Inkling (Xhigh)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"a8320f421bac97a284f8e6e159d09c3b8ea51df06774784c8240f3995ce9aaa8","result":{"confidence_high":52.756663,"confidence_low":41.304043,"sample_count":288,"score":46.990741,"score_display":"47.0","source_stated_rank":null},"run_fingerprint":"49246e2d45170bbb676e0ed62a1f1d848a3f2c8457e79c3f6e3db3ae72e689dc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling (Xhigh) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3035968fc7f1eebb6a46ad3a6b3934ad85a3af130b55f5a3c05a1e4e31008edc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.202548","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"inkling-max","source_effort":null,"source_model_version":"Inkling","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"15a3f86976f978661efbb94596401133b13178dd3ffac3599cb852845d2f49c8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":946.0,"score_display":"946.00","source_stated_rank":52},"run_fingerprint":"49619ecaedf10a5bf2bab1303eea5801ff6ab21732b36a7e14b60c8e19014b0c","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5b46a048212c2bd56946fe13","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8a3993d51c6d477bc05f5641bc8278f10354091ff81ed4f50961f973b1120416","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.605949,"date_is_proxy":true,"latency_seconds":1105.492,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.659},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":256000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"bf684a59dfb6fe7dfd330345e6342d11fcc7f179953bd38ac2b8d41b9c625669","result":{"confidence_high":26.38564,"confidence_low":12.042359999999999,"sample_count":null,"score":19.214,"score_display":"19.2","source_stated_rank":79},"run_fingerprint":"b37306f8dfe92c1339a23a3e5568f45b0eaa515eb45f1377604b96ba0e5b0920","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_777456089accb2a956038d7c","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9830e396fd3ce8a41850c9f08c9955082df8c992af1b425c6613604d93f31f2d","metric_details":{"license":"Apache 2.0","variance":10.653102513765463},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"inkling","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b770f209b1e64e4a186e8334054173830d4fe85be98e7ee8cf10997ae887b9a7","result":{"confidence_high":1418.9974076013962,"confidence_low":1406.2031190913985,"sample_count":13705,"score":1412.6002633463975,"score_display":"1413","source_stated_rank":81},"run_fingerprint":"46d5e8c71d3bc0833717be7e52456ac99588d4d0fc6365970e77e91a90b0d415","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_75b2751fe3dec6d639534bf1","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"352b567e4dd74610dd1b4d8f9aabacd9650cfa5e460553750ecdf289679c721a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"Inkling","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"fa0af402a7f7765ca34cca98c2d6e935da48f64f3172d78b7f20731525a19eed","result":{"confidence_high":1417.32,"confidence_low":1402.65,"sample_count":8970,"score":1409.99,"score_display":"1409.99","source_stated_rank":80},"run_fingerprint":"31b3457936b89a5b2cb112f7ceff4e8bcce0510a610337a22d5d3ccf7b2ebba2","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2145dff0d202c295837e91b8","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e42005369dfe78422d0fe168a8bbfcde8e6e62c5ddcd577e73156e14d737bd2e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-15","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"Inkling_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"4640d6de029d479249606816cda2c2b788d7aee89ef4aed6716d30abc6d774f9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.29,"score_display":"32.3","source_stated_rank":146},"run_fingerprint":"6f23d793954b9382d615cfe1f938c3e5f3cf248661ecef00cf2498b1ccfa1d80","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a026da84feee358d3fb40d36","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c2f7e26b6c40749b894333ab1da6786885364a298ad6c0ad3285fe45d7250703","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.630691,"date_is_proxy":true,"latency_seconds":771.551,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.375},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":256000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"0.99","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":1,"upstream_model_id":"thinkingmachines/inkling"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"0b2c1c9eb350050ad021a4cf9f8ecb95fa274c01daf642188755515257b9e840","result":{"confidence_high":48.301,"confidence_low":46.831,"sample_count":null,"score":47.566,"score_display":"47.6","source_stated_rank":64},"run_fingerprint":"c2d3ca027632010f4738b6ad0b8c31afb422e1f2ed15b9241c02d2bbedfdd78a","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_551396cfb7201bd63ba1e871","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"0.99"},"provider_mode_key":"0.99","raw_label":"0.99","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9436985d8072495209546f9ef52e80ac02510148adab9650fdb2495eee47f72e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-15","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Inkling (xhigh) · terminalbenchV21"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"xhigh","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"inkling","upstream_model_label":"Inkling (xhigh)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"a3cafb062775e58f2c5ea830f9fd7e9154ffebafdc989c95888605cc26b5c70c","result":{"confidence_high":64.967783,"confidence_low":44.726148,"sample_count":89,"score":55.05618,"score_display":"55.1","source_stated_rank":null},"run_fingerprint":"b458dc27d9015274e32fafb079faebc9e432ba2b11798fa655cb8136160bb600","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling (xhigh) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":73,"at":"2026-07-24","basis":"evaluation_at","evaluated_at":"2026-07-24","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f87afbb821ce0220345db8274602b3a6672c66a9df2252e05c18ee2b58b4b5eb","metric_details":{"cost":null,"pass_2":16.666666666666664,"pass_3":13.402061855670103,"pass_4":11.34020618556701},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"inkling_sierra_2026-08-04","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"9b3a9ea0b94586db52813b863262b4d8c3a015bf4bc6452b53ac000c57e1562f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.0,"score_display":"25.0","source_stated_rank":null},"run_fingerprint":"8760f15e687279704b2f7119d6f5f251daf58ef074169c7d9720d0e4aef31e14","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":6,"is_provider_default":false,"normalized_effort":"max","profile_id":"profile_5b46a048212c2bd56946fe13","provider_config":{"source_label":"max"},"provider_mode_key":"max","raw_label":"max","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T02:30:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T02:30:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4c4e7ff57fbdc3fbb87fd5488141f6b0df0067cf59a239dae8d932c01a5b33c6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-15","source_locator":"Leaderboard models table · Inkling (Xhigh) · tauBanking"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"xhigh","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"inkling","upstream_model_label":"Inkling (Xhigh)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"4e45fccab2f0c684a35ece451d1e59e07c9478d1aafe1468c4381f98d7054fcd","result":{"confidence_high":38.768255,"confidence_low":20.970583,"sample_count":97,"score":29.072165,"score_display":"29.1","source_stated_rank":null},"run_fingerprint":"7417a30821110a9a71d9a4a5195f6bcffbb266d743d71d53fbc914201636b753","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling (Xhigh) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":81,"at":"2026-07-15T18:19:13Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-15T18:19:13Z","status":"fresh"},"measurement_id":"bf3a44f7ea95c01157f80ca594f23057f9866c89a4c5e4afe067d15ffddb5834","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":2.6,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=Inkling (xHigh); createdAt=2026-07-15T18:19:13Z","source_row_created_at":"2026-07-15T18:19:13Z","task_pass_coverage_threshold_percent":75},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"Inkling (xHigh)","source_reasoning_label":"xhigh","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"e16603905daa647f1b5c21a8c8c347f4b30d024df41b487445769806a327593c","result":{"confidence_high":78.6,"confidence_low":73.4,"sample_count":1000,"score":76.0,"score_display":"76.0","source_stated_rank":3},"run_fingerprint":"81763b83d490e8bf07de77b66733ca3cbbd9354dd18af6264ebe35f8b9b460bd","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=Inkling (xHigh); createdAt=2026-07-15T18:19:13Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1f49424383f22ac869093f21841c6b8c042d4f38a783e0386a7d3afb820c1ad3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.488","mean_standard_error":"4.8","model_release_date":"2026-07-15","notes":null,"standard_error_points":490.00000000000006,"upstream_source_url":null},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Inkling","source_effort":null,"source_model_version":"Inkling","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"af2a8fe70d500fa9a45ec3ee992de9d4e7b6d47dca9ac8523c3eff36a94012d8","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":33.8,"score_display":"33.8","source_stated_rank":37},"run_fingerprint":"aef72d341e350bc29878f50e53b6a393965b58d9a8770a801cb01d14d54a7728","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2c3ca31d2106579e73b85c2c","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"219fcbddc3aad5d9fc59481b2a078c863b084829223a5666fe2b43ce0a5ffe06","metric_details":{"confidence_level":0.95,"license":"Apache 2.0","session_count":42813},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Inkling","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"37c10509e7a704ce305aee1f573f32060d7b0237092d2d93e97d23c58fe520f4","result":{"confidence_high":-0.09817640370218436,"confidence_low":-0.11902091761974418,"sample_count":1798990,"score":-0.10859866066096427,"score_display":"-0.1086","source_stated_rank":48},"run_fingerprint":"d15ece58d90ccf79094e12f550fccc1e1ef1fe67644087c73427e9851131e0d2","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_868910b5371c3f0911dd2a6d","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":2,"at":"2026-10-02T19:14:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-02T19:14:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4916661cb0e91873fe6bf09854558ec94a8bc716a21cce7682b4fa82cebddfa3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-15","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Inkling (Xhigh) · gdpval"},"model":{"id":"thinking-machines/inkling","name":"Inkling","provider":"Thinking Machines Lab","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"inkling","upstream_model_label":"Inkling (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"c803e6d3d1c67ddbfee8640f0ccfc5a84627a19a3c821bc6026f1bbfdd2914d7","result":{"confidence_high":1102.48,"confidence_low":1055.86,"sample_count":220,"score":1079.17,"score_display":"1079","source_stated_rank":null},"run_fingerprint":"1ffe1a9b903655cafe90babaf65b05fb90c51b540ca25f181306a511bb6fb066","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Inkling (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_d7ea10543bcfd3f2dca6a744","provider_config":{"source_label":"xhigh"},"provider_mode_key":"xhigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2bbc38e4f2fcaa1be63abbfbe82ecb39d09cf376ac2683d5715766a7938fe325","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-07-21"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 3.5 Flash-Lite; model release: 2026-07-21","source_accessibility":"API access","source_model_name":"Gemini 3.5 Flash-Lite","source_model_release_date":"2026-07-21","source_model_versions":["gemini-3.5-flash-lite","gemini-3.5-flash-lite_high","gemini-3.5-flash-lite_low","gemini-3.5-flash-lite_minimal"],"source_reasoning_efforts":["minimal","low","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9e51e3fe7be0797f46ab59c49547c2dbdad686319ed3b9b859b1e8f16fceba44","result":{"confidence_high":146.69,"confidence_low":142.46,"sample_count":null,"score":145.11,"score_display":"145.11","source_stated_rank":82},"run_fingerprint":"571a21e2131caf7949c46c443b2e248b10c05ea2c9e54702c82dbf448f2718ae","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f87c6a70e804aea75a8cc3f0","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3057bad137ee82bf22bc40c2b2166aba6c15edfb6e60d3cf51c3ecfa0cc61789","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · intelligenceIndex"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-5-flash-lite","upstream_model_label":"Gemini 3.5 Flash-Lite"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"bf7e3dfd59300f1955c2e0d0e66504e018e2cbbd1569e814c5b147c0d8f63b9e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":22.168542,"score_display":"22.2","source_stated_rank":null},"run_fingerprint":"68c10cc4e7093c144914c43607c10f3ec0ead4c5710d4de28fb62d3bd61cae5d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b05f8827831fc092f12f06f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"54fc35cce4c21c69c7e516ad1c784ecd2e3facade759dddd6d84486c82ac8564","metric_details":{"license":"Proprietary","variance":5.237481868730745},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1459.4518487822813,"confidence_low":1450.4808798440092,"sample_count":35763,"score":1454.9663643131453,"score_display":"1455","source_stated_rank":71},"run_fingerprint":"aa80aad00f476a0c2320d13e5ab01890f47ccd5de7f01ac36069647059c7174b","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bb75ca61c28fd70310b7ca1d","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"641ab6f107a1299f0bb18bd93af7138dcd4777eab75082f36da1495daf51b94b","metric_details":{"category_scores":{"Agentic Coding":45.25266666666667,"Coding":76.0715,"Data Analysis":53.24966666666666,"IF":67.23774999999999,"Language":71.82033333333334,"Mathematics":73.73949999999999,"Reasoning":60.1875}},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":63.9369880952381,"score_display":"63.94","source_stated_rank":65},"run_fingerprint":"77d9cddc17fac61fcd65526ca327373b3c75b6e0852225f48ade2b8f059c543d","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6ec9ae605e25c381696f40eb9e3a1550faf347c09f6e043d562ca5ac7924693c","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · hle"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-5-flash-lite","upstream_model_label":"Gemini 3.5 Flash-Lite"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fb1640114ef18ae7180c221113a08f160d24cdb7603120ba23a9a268dcc2943c","result":{"confidence_high":20.517556,"confidence_low":17.220712,"sample_count":2158,"score":18.813716,"score_display":"18.8","source_stated_rank":null},"run_fingerprint":"187078e33c9ea4edbb1c949c75cceb51febea3e467e6a3b86c7dcd28d9813e3a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b05f8827831fc092f12f06f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-23T02:56:47.235286Z","basis":"evaluation_at","evaluated_at":"2026-07-23T02:56:47.235286Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b7b48e5d4e8a5d1c7537e5d35474222a313fdeca249f5df806a17a45eace371e","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.5-flash-lite","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"82f77751ebf4e0e1bdb747f60520ab845ac1968d26418478e88298d4c1f1b07d","result":{"confidence_high":35.93020400954324,"confidence_low":30.232176222428514,"sample_count":null,"score":33.081190115985876,"score_display":"33.1","source_stated_rank":21},"run_fingerprint":"6d29afe28493c991be8e6c70fcfb5781afb18d3900394673d0d2e63f5bb51d1f","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_995ea331927f1b2b53b092cc","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"36b5f5772d800f58105c41ae62de244e0d07cbdcf8e611695b7e18e08e9416cb","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.01235,"date_is_proxy":true,"latency_seconds":13.848,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.847},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":81.818,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash-lite","zero_shot_accuracy":85.859},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"6872002b8f7c8bacb3fab4fbf6dc791d40a8662a29d25339b4bc5723f32ccf15","result":{"confidence_high":87.45812,"confidence_low":80.21788,"sample_count":396,"score":83.838,"score_display":"83.84","source_stated_rank":56},"run_fingerprint":"709840ae20ec386296a7f418001a887a6150620371bcae0f7f97f12a5c2e6ba1","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"518155585799087741ef6b1be3e5ad1e6bee22feb0899663f81bcf06e5acce77","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · gpqa"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-5-flash-lite","upstream_model_label":"Gemini 3.5 Flash-Lite"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"df559c2295ba4da18a09fa7f93cb464fff40b7b6de0bcf7b51f8868660cb5520","result":{"confidence_high":88.313279,"confidence_low":78.075414,"sample_count":198,"score":83.838384,"score_display":"83.8","source_stated_rank":null},"run_fingerprint":"9a5a82f3b46f0cc69e931d487546ee561b6ce33913842585a2120b5a14639bb2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b05f8827831fc092f12f06f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:35:04.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:35:04.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b6265992d877e09b41f553966ff8a749cbd274858185c34069ffce14378f39ae","metric_details":{"best_score_across_scorers":"0.8333333333333334","model_release_date":"2026-07-21","stderr":2.6552207828215257},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"chbh6XstiqtUa5W9zuyfjS","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fchbh6XstiqtUa5W9zuyfjS.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/chbh6XstiqtUa5W9zuyfjS.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"0aa513b4179a85f734b8211aba5ec93542dd3baf5db1cf74dd77b24c83d22db2","result":{"confidence_high":88.53756606766353,"confidence_low":78.12910059900315,"sample_count":null,"score":83.33333333333334,"score_display":"83.3","source_stated_rank":104},"run_fingerprint":"9a4d291b89dd350310a5c54c3e87c8305691a48e6e55ac7eec3dedaa81960a45","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:35:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:35:12.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"10b7b4e3c79d5218fc4877c5cfd717d8eafcbccb24a9a3f12c396e9daddcf4fc","metric_details":{"best_score_across_scorers":"0.7575757575757576","model_release_date":"2026-07-21","stderr":3.053289223393202},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dgSJTFQ4UtuQKEhB9ss6K7","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdgSJTFQ4UtuQKEhB9ss6K7.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/dgSJTFQ4UtuQKEhB9ss6K7.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"4a2e0e52fe2955c99bd3853eb60a7972d74a8a3c7ff464966283bde7ad49529b","result":{"confidence_high":81.74202263542642,"confidence_low":69.77312887972508,"sample_count":null,"score":75.75757575757575,"score_display":"75.8","source_stated_rank":152},"run_fingerprint":"831ef986e6d31de5eea79e5af463c96fc04ce19da3bdad35fda34f507dac546b","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b798c6a4a17b37484fd2cc8a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:35:11.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:35:11.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8d954a7afbdbe3c9b450e0bc3bd2f79f548472191a35a804e200fa1af403cb1b","metric_details":{"best_score_across_scorers":"0.7424242424242424","model_release_date":"2026-07-21","stderr":3.1156269519646824},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7aAKcnsCXoSumfAu8mr2un","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7aAKcnsCXoSumfAu8mr2un.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/7aAKcnsCXoSumfAu8mr2un.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"bbff8f836bb49773d5c29b83306a9881f1ac7e50300c6724be8c64ec5813182c","result":{"confidence_high":80.34905306827503,"confidence_low":68.13579541657347,"sample_count":null,"score":74.24242424242425,"score_display":"74.2","source_stated_rank":162},"run_fingerprint":"71d4f79a2c0fe1cd1cf9ba1428cd0cb3cd2921624748e29de8d2f45b27316a3c","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_f96cd64c3c0ba13e927639f7","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":74,"at":"2026-07-22T22:31:14.559747Z","basis":"evaluation_at","evaluated_at":"2026-07-22T22:31:14.559747Z","first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8829f89001c653c945a38af43c0261dbbe7057a87c104ec7e1b94af9265e1f2b","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemini-3.5-flash-lite","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"731ed476ecceeb9903703b48098b5bdb7f3a97a000e01e62ad5a68f251a83888","result":{"confidence_high":78.93065338520586,"confidence_low":66.52389206933961,"sample_count":null,"score":72.72727272727273,"score_display":"72.7","source_stated_rank":26},"run_fingerprint":"215aed282237db33622d212c454b86216d67885d40ce7dc1418e5b791de22d4a","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5fa766baefd21d4bbadf3aa4","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"50ae22067c7de1b42c8af3880e89eba7331e5c9609c78fbb4423640a1b677978","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.004522,"date_is_proxy":true,"latency_seconds":4.823,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.344},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65536,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash-lite"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0e95e6cc8c1f0a25c57667c901be5b6d0cbfec19a3b54c93724a33a30f099707","result":{"confidence_high":86.51024,"confidence_low":85.16176,"sample_count":null,"score":85.836,"score_display":"85.8","source_stated_rank":54},"run_fingerprint":"78a52c0f86fd45ffc67102bf98fac600ec5214d93a1c6b6508dee02966a213f3","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:35:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:35:36.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7f79542289b15eadbb931a974ba35609a410717978718b20f20cdd4d75b785cd","metric_details":{"best_score_across_scorers":"0.7111111111111111","model_release_date":"2026-07-21","stderr":6.83294324254051},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mqsc4vFuKtAHXxbYyX9WwK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fmqsc4vFuKtAHXxbYyX9WwK.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mqsc4vFuKtAHXxbYyX9WwK.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f7647352e260863b823a88cb30a738a5bb8bc76b558091650db4b364c51f5230","result":{"confidence_high":84.50367986649051,"confidence_low":57.718542355731714,"sample_count":45,"score":71.11111111111111,"score_display":"71.1","source_stated_rank":132},"run_fingerprint":"7ce86ed020ee7070d6dc58a91f38c4f2f8e7a13b16e151e9c59fd88eab38b899","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:36:02.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:36:02.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6f35db2fbb7c81381e8431e63c53d8a377a80197f5d6374a5426543ce4661e50","metric_details":{"best_score_across_scorers":"0.6","model_release_date":"2026-07-21","stderr":7.385489458759962},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"B3Kno3buYWRYbZxU33STiX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FB3Kno3buYWRYbZxU33STiX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/B3Kno3buYWRYbZxU33STiX.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0911c18ed69d2b99fbf4847dae0e36571e98f996265109257ae428c01beb0a56","result":{"confidence_high":74.47555933916952,"confidence_low":45.52444066083048,"sample_count":45,"score":60.0,"score_display":"60.0","source_stated_rank":164},"run_fingerprint":"ce750abd863bf5e684af18caa314386fe3dd71c09ff15287bbd288edc3515782","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b798c6a4a17b37484fd2cc8a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:35:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:35:57.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c99de8f407aeace17212aa2e9728e3bd7cb6cca097c976f4c8c6a38c58ab93cb","metric_details":{"best_score_across_scorers":"0.5111111111111111","model_release_date":"2026-07-21","stderr":7.535922203472521},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ivr2LfmdZoCMfLME7jh2L8","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fivr2LfmdZoCMfLME7jh2L8.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ivr2LfmdZoCMfLME7jh2L8.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"650fef77e263d21087d1a21e07068f9f131fd8999c6f107d7ff7861128087fd6","result":{"confidence_high":65.88151862991725,"confidence_low":36.34070359230497,"sample_count":45,"score":51.11111111111111,"score_display":"51.1","source_stated_rank":187},"run_fingerprint":"81746c265e122065b06e88080fc606f226e9f9578d8f16cf252ddd6f233fd10c","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_f96cd64c3c0ba13e927639f7","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T01:20:17.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T01:20:17.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bc9232605c5ca178f804475c55de6e0b298aa0e0d05179f1dad579fbe16c8113","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.2596491228070175","leaderboard_private_problem_count":285,"model_release_date":"2026-07-21","public_example_count":10,"stderr":2.601675082238553},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"8o2J9uQLKDCTiCFwYdkJRK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8o2J9uQLKDCTiCFwYdkJRK.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8o2J9uQLKDCTiCFwYdkJRK.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.5 Flash-Lite","thinking":null,"upstream_model_id":"gemini-3.5-flash-lite_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"7243a9334d237e65559a5eccb07320f679a3321dc0cd08c2f02302f8aa71cddf","result":{"confidence_high":31.064195441889318,"confidence_low":20.86562911951419,"sample_count":285,"score":25.964912280701753,"score_display":"26.0","source_stated_rank":75},"run_fingerprint":"3e6ec68353a4270fb289416d65e3d47401399560e1f7c1111c8ca4107413020e","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"565a34d4381092cd5a0b8aafb99fed906c7929287407f9db19a7a2af7878b60a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.13899638250000004,"model_release_date":"2026-07-21T00:00:00.000Z","results_url":"https://arcprize.org/results/gemini-3-5-flash-lite"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-5-flash-lite","model_type":"CoT","upstream_model_id":"gemini-3-5-flash-lite-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7b05fe75cdb7f2a11f099e9b0f6748abdaa81040fcc26e106ae9ac77eea282ce","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":10.277777777777779,"score_display":"10","source_stated_rank":143},"run_fingerprint":"215be8736b58ee935e05620f28e0cde26784c7edc0cc427c3b539d4a13dc7986","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9e36c5207369e86fa78d001027a0d18cb22a45d2da82e05b7023e5d22f6f95d9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.04367807000000001,"model_release_date":"2026-07-21T00:00:00.000Z","results_url":"https://arcprize.org/results/gemini-3-5-flash-lite"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-5-flash-lite","model_type":"CoT","upstream_model_id":"gemini-3-5-flash-lite-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ab306791d4570222f059185d07212e629453c798b957a711b7945d289f5f041a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5.277777777777778,"score_display":"5","source_stated_rank":162},"run_fingerprint":"91164815cfd30245220f68363e565bd7ce65cd7b54d94ad9916d22bcacd92374","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_4a7108387c4db162d35f3d00","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dc863824b18a3bfb7ef1551f63ceaaae4cf5e910ecce9f096e3394dd437f91ff","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.017051445000000005,"model_release_date":"2026-07-21T00:00:00.000Z","results_url":"https://arcprize.org/results/gemini-3-5-flash-lite"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-5-flash-lite","model_type":"CoT","upstream_model_id":"gemini-3-5-flash-lite-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"47698db7dbc7fa876c06900f0f25cb73deb28066ef5b190d2b33d42857d3b1a8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.5277777777777777,"score_display":"2","source_stated_rank":210},"run_fingerprint":"6bc693f74d5bcc5e3b82f31897adf42ff033cff2a5214cae7ef9d7c6d98fa38e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b798c6a4a17b37484fd2cc8a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"36d9eaa67cc1c2457b1e640145a7081c71667400bef0a66e6fd1dcfe3cb20a68","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.014339924166666665,"model_release_date":"2026-07-21T00:00:00.000Z","results_url":"https://arcprize.org/results/gemini-3-5-flash-lite"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-3-5-flash-lite","model_type":"CoT","upstream_model_id":"gemini-3-5-flash-lite-minimal"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"048ea7181ef973305f76bd73a9c03852ac8b984607994b25a6e8dc18763aca3e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.8333333333333334,"score_display":"1","source_stated_rank":226},"run_fingerprint":"f0a6157cf2684e41aa238e0cccebcc4fd70e9fd18d8e81708df8de69594dd799","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_f96cd64c3c0ba13e927639f7","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"18f4f9aeb8198dd3d877cadfe22200c3de5e32dfd0ff509135b89bb0f4997263","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite (High)","source_effort":null,"source_model_version":"gemini-3.5-flash-lite_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"d1f2992d0e736a9ac6801320d3539f54cf0dba1f86ee73e5046f42a0eb341531","result":{"confidence_high":13.845565,"confidence_low":7.5488,"sample_count":360,"score":10.277777777777779,"score_display":"10.3","source_stated_rank":139},"run_fingerprint":"0d715278172320e56029726b1d175f2871ddef8a809cdbc8b26b1af8cbb00e27","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"75c701f78b8e59388115b7761562adf1c5c5b885b243b086de19926e61c4bd22","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite (Medium)","source_effort":null,"source_model_version":"gemini-3.5-flash-lite_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"572e0114fb268c98734a84b27aa5d6182fc27668f6fcd91dce9687d0a2d047e5","result":{"confidence_high":8.095476,"confidence_low":3.404474,"sample_count":360,"score":5.277777777777778,"score_display":"5.3","source_stated_rank":158},"run_fingerprint":"cf3fb17ab712e2b7ef9a84a56aa69c3a4b5b94042335d1ddd10f54cc80395da7","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_4a7108387c4db162d35f3d00","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"20cbca8e0bbc80b1abdaa5ca30bcd897bd5e6c6d3d5458f39e1187009263e22e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite (Low)","source_effort":null,"source_model_version":"gemini-3.5-flash-lite_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"73e420534f882f25e4c86da43d2dc7da6853f0749cd0eed21dff26aba3a73f74","result":{"confidence_high":3.399856,"confidence_low":0.679282,"sample_count":360,"score":1.5277777777777777,"score_display":"1.5","source_stated_rank":203},"run_fingerprint":"8cf9ed43439fe967056428f6e9d3652774d890f5708da42484d1fbcf16192d56","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b798c6a4a17b37484fd2cc8a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"46c7f87dc8c63d47e6dbb1f3d476898ffe76e001c2a84008b85651a07d6bf8a4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite (Minimal)","source_effort":null,"source_model_version":"gemini-3.5-flash-lite_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"15874f7458dd37b3c57d08b12dcd0c6fe20c15fa2a6096b90084e3cf8d508b9c","result":{"confidence_high":2.421113,"confidence_low":0.2838,"sample_count":360,"score":0.8333333333333334,"score_display":"0.8","source_stated_rank":218},"run_fingerprint":"7cfda97281d3011d588dec27fc2d9561a4ff1ce0998fd5d11181a74a61c303e2","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_f96cd64c3c0ba13e927639f7","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T01:20:17.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T01:20:17.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ad32554392d81ea009ccf4f70b6090553883d5edb4e848b065b848056bf4fdeb","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.0","leaderboard_private_problem_count":41,"model_release_date":"2026-07-21","public_example_count":2,"stderr":0.0},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"juNVfKBcYTwaKnk9GRA2zL","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjuNVfKBcYTwaKnk9GRA2zL.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/juNVfKBcYTwaKnk9GRA2zL.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.5 Flash-Lite","thinking":null,"upstream_model_id":"gemini-3.5-flash-lite_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"34ca0cd997d4eb6e78ce323fa6abe83669f340371db64e91ba8daa4505c9d13f","result":{"confidence_high":0.0,"confidence_low":0.0,"sample_count":41,"score":0.0,"score_display":"0.0","source_stated_rank":66},"run_fingerprint":"9ad4fe7746e4d05c474768157735c09660feb97510d3f6cca36d99c748b616f4","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fe434de189bb559a4cb99066c059c9dbec493d7cb96c0eadac89d475becfa89b","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.006577,"date_is_proxy":true,"latency_seconds":7.799,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.891},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65536,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash-lite"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b0345c1f662dac4063166d79495f5cd4e4b25a6b47abd3fdd03d35e2febbf8cd","result":{"confidence_high":85.33036,"confidence_low":81.83764000000001,"sample_count":null,"score":83.584,"score_display":"83.6","source_stated_rank":30},"run_fingerprint":"45dff96feb9631850e5ecc92171799973ee89d7261507f2c2e66ebf66ea4e46d","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"62dd3c26ae1e8d6a2608b0e3bd4c4608f34664b0d920c6f52f215693ed7a4953","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · mmmuPro"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-5-flash-lite","upstream_model_label":"Gemini 3.5 Flash-Lite"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"40a1967d859eada8ff515984bb4a921a3b6a43528233a13b9b232f182d0e17b5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.017341,"score_display":"79.0","source_stated_rank":null},"run_fingerprint":"9d873de09ef3c242a00cbe160189f3e768732e9bad480dd2474f2862b243c7f5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b05f8827831fc092f12f06f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9d467073be1714f16306d5fd8beaf05eb762882fac24dd35a58ae152554ee724","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite","source_effort":null,"source_model_version":"gemini-3.5-flash-lite","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"90ca8e52c2b63e1b9c824fb5be3432396b801496d2934ef3614d355d44d2fb22","result":{"confidence_high":5.132974,"confidence_low":0.0,"sample_count":71,"score":0.0,"score_display":"0.0","source_stated_rank":167},"run_fingerprint":"b8ad86f81392c00eca9fa25835f1107dcce460f0c637ae2b7199eb1c2cee524a","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e2c5d1a38c353372deb60611","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e93b5c79b5b8987bb26c408ef303e788bde0c1972ee39d587d2e9a69dd2e53b2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · critpt"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-5-flash-lite","upstream_model_label":"Gemini 3.5 Flash-Lite"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"1d5e86c7827e32c4498f46dfb2138486c0a77739b9b345cdbd24d82850cf93d9","result":{"confidence_high":5.202487,"confidence_low":0.0,"sample_count":70,"score":0.0,"score_display":"0.0","source_stated_rank":null},"run_fingerprint":"692c3756f84937b187faf31a9ded8d919f3d75a52b102b55d51c543b096aa419","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b05f8827831fc092f12f06f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:35:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:35:24.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2f216ea9dfe91aa509c7903cec321daa344dc7edb21e1b1c759a70ec53ff589b","metric_details":{"best_score_across_scorers":"0.22","model_release_date":"2026-07-21","stderr":4.163331998932265},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BSSNShjap29bDUHz4gA9oh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBSSNShjap29bDUHz4gA9oh.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/BSSNShjap29bDUHz4gA9oh.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3812b6d3cda6535638a92ea20dced438d8d3be35157a2cb594646eedee8d0c52","result":{"confidence_high":30.16013071790724,"confidence_low":13.83986928209276,"sample_count":100,"score":22.0,"score_display":"22.0","source_stated_rank":76},"run_fingerprint":"81e83b47504af9a39803233f4a91c69bea10fb4e9ace8d109452f9dbcc64fb37","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:35:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:35:36.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d0d8a0d654824ea0972b7d232d95e49f8c328af2c3c3afce75cb6798e1132d08","metric_details":{"best_score_across_scorers":"0.18","model_release_date":"2026-07-21","stderr":3.8612291966536914},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"i62SEaSpi6GbBmWqEDSDmV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fi62SEaSpi6GbBmWqEDSDmV.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/i62SEaSpi6GbBmWqEDSDmV.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"45429b76ffd54f019ef80b8785375e87ec99e06935e40e0c1c8442bf9dfd2a13","result":{"confidence_high":25.568009225441234,"confidence_low":10.431990774558766,"sample_count":100,"score":18.0,"score_display":"18.0","source_stated_rank":101},"run_fingerprint":"038b0d9631161a38eb421994cd878e74dc9a345e33cdf91ddf75b9f6cda353b0","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b798c6a4a17b37484fd2cc8a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:35:43.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:35:43.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4af4a2311781a04eeda07945def830158a37378bcb4f28458e7e0add8f4f357c","metric_details":{"best_score_across_scorers":"0.21","model_release_date":"2026-07-21","stderr":4.093601807403323},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"X6WHjwmjSHe7pF8R7F3zsT","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FX6WHjwmjSHe7pF8R7F3zsT.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/X6WHjwmjSHe7pF8R7F3zsT.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"09a309d1492ce1318fc4e551a875d2f76164e33d2bf168c026e5def1d235bb04","result":{"confidence_high":29.023459542510516,"confidence_low":12.976540457489486,"sample_count":100,"score":21.0,"score_display":"21.0","source_stated_rank":80},"run_fingerprint":"1d5f1a8da8432e266699e69c69efe621c3f73fe5cdbc24cd4e813808affa03d1","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_f96cd64c3c0ba13e927639f7","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a38c98f1208e0e8490d427e578bdd226bfc8e5de03b1e58faf933736d1a8a62c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite (High)","source_effort":null,"source_model_version":"gemini-3.5-flash-lite_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a393ad5a030262b708816637dea83dd77cc9a95eeaa58236c8ac0eaf5ce4da59","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.5,"score_display":"53.5","source_stated_rank":164},"run_fingerprint":"0d8878dc450bb259ff3b8a4b6c425d1d7cd230590b7afdb36ff60788f5402ad4","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"50ea7b003d61444b62962889ac55dfff0da77830d325a3114cdc1ad89e6ee189","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite (Medium)","source_effort":null,"source_model_version":"gemini-3.5-flash-lite_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"9237791a8c93e189452d38048c0ccf3554839ef9665aa7cc8c97e2d43d809836","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":32.33333333333333,"score_display":"32.3","source_stated_rank":199},"run_fingerprint":"0a103cbabc78d218dd42215893b9effa6dacef8de2ba525f9d4733d1f1ab20fc","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_4a7108387c4db162d35f3d00","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2f7ea0289c01e96d5f8162245504dfb439695eb7c5725d2cfad6b79f75ddb789","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite (Low)","source_effort":null,"source_model_version":"gemini-3.5-flash-lite_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"be253deb07527f792d652cac61c101e9ba179c42af45bc3dabe3948221442e73","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":17.0,"score_display":"17.0","source_stated_rank":230},"run_fingerprint":"55c811cb2a61db0a0e2e3c4b0617319936c456acfb19e9695f56777ba4d9a886","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_b798c6a4a17b37484fd2cc8a","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e28ee02d658c9933236b40705b656be88f12c84123d7065413c5ea781700a668","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite (Minimal)","source_effort":null,"source_model_version":"gemini-3.5-flash-lite_minimal","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"9bf302e1bcf8c3f46d99d8bcff7c8380fef6780cddabdc259e8e5dbbfb052e7d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":7.5,"score_display":"7.5","source_stated_rank":252},"run_fingerprint":"45ebcc2a5d65d6c42ea1ad3f37a49711d4e2af2ac368a22fb5ae1a566a7fb8ef","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_f96cd64c3c0ba13e927639f7","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a76c22177c024fea49331fce2d678ab5281b16ee3fbb03a750ab3ff89b5acd46","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.30898,"date_is_proxy":true,"latency_seconds":439.669,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.938},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash-lite"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"c200d52274ef04f976ba900a75b145e1ccae3e377e63fd7587240e5698a25ab0","result":{"confidence_high":78.79848,"confidence_low":71.20152,"sample_count":500,"score":75.0,"score_display":"75.0","source_stated_rank":44},"run_fingerprint":"aefc6267b7e56518b0eae8af7657084379f13bfa5b81d6724534e65c82b0e565","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_bf1ea14003c917dab90422d7","split_or_window":"vendor-public-own-harness","unit":"percent","version":"SWE-bench Pro Public · Google provider run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":76,"at":"2026-07-21","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:32Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-21","status":"fresh"},"measurement_id":"063186519f4e6eb51d53cb6de00bfedd749023ad33de8f9f8b52b4d002f68859","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/","evidence_sha256":"6da66e353db277e7170ea4373a8db36976625d453b6c657e06615bc49a2e8885","kind":"reviewed-static-report","published_at":"2026-07-21","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/","version_id":"a7f389d09d5782d0c608836b8f35530fc8295a18805ce9a1c9d67a31ccf68688"},"source_locator":"Model-card benchmark table · SWE-Bench Pro (Public) and evaluation methodology"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"high thinking; pass@1; default sampling; full public set; Google self-computed Antigravity evaluation","reasoning_effort":"high","result_published_at":"2026-07-21","results_as_of":"2026-07","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"44136d4d837ee8a028dabcd19190ce1339c941dd27de1d07382f153acaa6a039","source_published_at":"2026-07-21","tools":"software-repository shell and code-editing tools"},"run_count":null,"scaffold":"Google Antigravity evaluation infrastructure","tools_allowed":"software-repository shell and code-editing tools"},"protocol_fingerprint":"f3d9a58f4f491779939c699afe0b172d3a63cd8995a7e2f2f85e6db79b18f4c4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.2,"score_display":"54.2","source_stated_rank":null},"run_fingerprint":"e041a87b3bb863d2a54b43203460ab8c60d4a6c0771f46ea525bfcfad84944cf","source":{"content_hash":"0bef356c5c59500c79572ec9a41ce486756eae03ee882b2c1edf688a27194379","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/","id":"google-gemini-3-5-flash-lite-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · SWE-Bench Pro (Public) and evaluation methodology","name":"Google DeepMind · Gemini 3.5 Flash-Lite Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a539482a6cce92d8707665144782da9ba633ae4c97a5a5b003ba595d20e74fdb","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.019272,"date_is_proxy":true,"latency_seconds":21.084,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.098},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash-lite"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"85d3c0ae9460fcd1f73a0257e167932175fa658309c6f39ed17282d2cfa89345","result":{"confidence_high":81.16608,"confidence_low":76.86192,"sample_count":null,"score":79.014,"score_display":"79.0","source_stated_rank":75},"run_fingerprint":"7d3dda5efb5ca4ee0ffb44be5c60c3a9eeacbd8e676438e645c30d10da5b5ea3","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2e9efca8c0e26fd8479d9283e3ead061199552db740c523f8380754877806622","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.5 Flash-Lite","source_effort":null,"source_model_version":"gemini-3.5-flash-lite","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"5c18565f82b5f82a99e1759dba12b495cc395a3909714746dfb5548dfff967a2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.3194444444444,"score_display":"41.3","source_stated_rank":141},"run_fingerprint":"2b9c6fd965dd80472b61f9340cee9517bde685eb692924871672dfd3e8753668","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5ebede4cb3502b6a8f56583f","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ef532c24d7a2383b7bb74acf2f9d823509e0e13d2a1624e68d11f714391480aa","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · scicode"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-5-flash-lite","upstream_model_label":"Gemini 3.5 Flash-Lite"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"37d6574f9da7e132d6bd529d981521fabf88cc34939c64ce4fe5a58ee83ce20b","result":{"confidence_high":47.084323,"confidence_low":35.783095,"sample_count":288,"score":41.319444,"score_display":"41.3","source_stated_rank":null},"run_fingerprint":"16adb55715c62c2b1f7cb496127daf4bf14122c4ee1e7a10099526a9fc2a4b7b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b05f8827831fc092f12f06f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9a0076adabec116eb64bce45b2a17a2f49e797ad6b8822a24aa6d91b0f6c0f5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.045726","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3.5-flash-lite-high","source_effort":null,"source_model_version":"gemini-3.5-flash-lite_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"348938f38e06c3caecddd546cc73a045febe8ba29b77598e5161c8e6611427f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":765.27,"score_display":"765.27","source_stated_rank":73},"run_fingerprint":"f823a4b99eec8712592c901ecb8fb6df001fc1a3e404843b2d873001d9441f99","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b315fcdcf7f32b9f9e278d4fc09048cc6adfd44a92e3df45b64443bf95746c74","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.833111,"date_is_proxy":true,"latency_seconds":501.522,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.627},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash-lite"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"d4f58f1f7f2df429ace859d58e968d93f1ba961ae1f18da381200293020e8901","result":{"confidence_high":46.22992,"confidence_low":28.092080000000003,"sample_count":null,"score":37.161,"score_display":"37.2","source_stated_rank":63},"run_fingerprint":"571a54ec79d7db7c8a2c2d6746063d0bb7855c898dc95dba74bde12bf8817b5d","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f7868c995f42ac3861859c7f874f0d4fe2fd2a77c4f686f15a945f550f7db111","metric_details":{"license":"Proprietary","variance":428.0135130854581},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3.5-flash-lite","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"14924f19b6c17a7bb93694f2789abe885d06d98995358f20265b1576110c8d95","result":{"confidence_high":1480.279797810666,"confidence_low":1399.182417206397,"sample_count":242,"score":1439.7311075085313,"score_display":"1440","source_stated_rank":70},"run_fingerprint":"5e84922e9ca89cf7f3936e4e6282a2aab7cd069398ed70a578581256e849b552","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a7f1f18e36f8a6a9774d349f","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f77a5355020500513d1551664a76da9d73dbd2e7464766e228184f1bb5a8b6e2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.5-flash-lite","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"d85e7502f484485be6b0a6d06878c4baa9649cfc66a2846bc82663efb3b5d051","result":{"confidence_high":1489.11,"confidence_low":1404.13,"sample_count":228,"score":1446.62,"score_display":"1446.62","source_stated_rank":68},"run_fingerprint":"3747bb4c78396529ef1e7934640aea94de00a1896d7b3f453fd97d232e5892cb","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2c54b234f0c299393757dbec","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2a8fed95c3fb4dcbf862fd94b83b355ec1efc8e47202bf160b8f7aacd53add01","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-07-21","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.5-flash-lite_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"e4e6c29684eeb5a121905f301b3c516feb86f2bdf63f3e2b92e356eebc3124b2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.0,"score_display":"39.0","source_stated_rank":120},"run_fingerprint":"ab365ca075c168048a13f192b00b93f1f796a30f1685b8f3e3131aca77bbba17","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8f442349140d9824230ded85698c70bdc7cf6de9e48373b639f092ecde0597cc","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.337052,"date_is_proxy":true,"latency_seconds":416.383,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.991},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.5-flash-lite"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"80787ce3a0f289432928b20659a083e49fd75b61e935cef1e3a9060d9afa076e","result":{"confidence_high":52.12936,"confidence_low":48.24464,"sample_count":null,"score":50.187,"score_display":"50.2","source_stated_rank":59},"run_fingerprint":"bfd0d59fe159d9b2c9c283a157cc6b455f897ffa77da40c1edc25442331098e6","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0e07154cf144e834e45bb6c97b6171a0f49a7f03184995b11601eef81b1fc39c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · terminalbenchV21"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemini-3-5-flash-lite","upstream_model_label":"Gemini 3.5 Flash-Lite"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"238c63f2947deef01a179c1321cdd27e913471ab2881b3511a779b7f99562859","result":{"confidence_high":63.556902,"confidence_low":43.264751,"sample_count":89,"score":53.558052,"score_display":"53.6","source_stated_rank":null},"run_fingerprint":"24d76f2b6d3a98445972e40490549b7706a1f8df3429904b15be5d6ba44418e9","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b05f8827831fc092f12f06f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"mean_reward","name":"Terminal-Bench 2.1","release_id":"release_bde217160cbff9a7aeb3cce7","split_or_window":"terminus-2","unit":"percent","version":"2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":76,"at":"2026-07-21","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-02T12:19:32Z","last_confirmed_at":"2026-10-05T12:33:35Z","observed_at":"2026-10-05T12:33:35Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-07-21","status":"fresh"},"measurement_id":"7aef4f30fa9fa99e5961a67d855c7452ae4a483e82e8b6660dae8dc14089e82e","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/","evidence_sha256":"6da66e353db277e7170ea4373a8db36976625d453b6c657e06615bc49a2e8885","kind":"reviewed-static-report","published_at":"2026-07-21","reviewed_at":"2026-09-09","source_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/","version_id":"a7f389d09d5782d0c608836b8f35530fc8295a18805ce9a1c9d67a31ccf68688"},"source_locator":"Model-card benchmark table · Terminal-Bench 2.1"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google DeepMind","evidence_verified":true,"freshness_proxy":true,"protocol":"Google provider evaluation; high thinking; pass@1","reasoning_effort":"high","result_published_at":"2026-07-21","results_as_of":"2026-07","source_content_hash_method":"html_visible_text_v1","source_content_sha256":"44136d4d837ee8a028dabcd19190ce1339c941dd27de1d07382f153acaa6a039","source_published_at":"2026-07-21","tools":"terminal agent toolset"},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"e6db65fd38541717a83790cbf22310b6ae7d8fad1f99f3c70c68f3bcb31ec973","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.0,"score_display":"54.0","source_stated_rank":null},"run_fingerprint":"15e954a514d63e9f8d8537c28d759df163e2847471bf365598a36146e331ca7d","source":{"content_hash":"0bef356c5c59500c79572ec9a41ce486756eae03ee882b2c1edf688a27194379","fetched_at":"2026-10-05T12:33:35Z","first_fetched_at":"2026-09-30T18:53:25Z","homepage_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/","id":"google-gemini-3-5-flash-lite-card","last_fetched_at":"2026-10-05T12:33:35Z","license":"Copyrighted report; minimal factual score fields with attribution","locator":"Model-card benchmark table · Terminal-Bench 2.1","name":"Google DeepMind · Gemini 3.5 Flash-Lite Model Card","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_4ffb600ecd36bbcb9902a5ac","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b913ecf076d4ce23b185ecc14f7ad47a1a912f0e82cd6630772513233a4ca11c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","source_locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · tauBanking"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemini-3-5-flash-lite","upstream_model_label":"Gemini 3.5 Flash-Lite"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"1c7da1345a51b831c7fcbfbd5257c8cfd66a5645da1e30539b5a69c10b0b3acb","result":{"confidence_high":26.285821,"confidence_low":11.239961,"sample_count":97,"score":17.525773,"score_display":"17.5","source_stated_rank":null},"run_fingerprint":"9fd86465119e19ce996fda3c9e38dfaa6994ed9308226f5f0249342eea3aff02","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b05f8827831fc092f12f06f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9366460132a64504b547b178737fdfdd9cd37775a6536c095fc7bd42089049fb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-07-21","reported_confidence_interval":"-27 / +27","source_locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · gdpval"},"model":{"id":"google/gemini-3-5-flash-lite","name":"Gemini 3.5 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-5-flash-lite","upstream_model_label":"Gemini 3.5 Flash-Lite"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"428c4f97ce86a090221f02cacdc63d5e4fb66aa3f5e14c0caee8d7f9780d50c2","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":969.52,"score_display":"970","source_stated_rank":null},"run_fingerprint":"5c8ae10fa322e30b4d6b73171164aafcf94a942a0051420fa9a2d90e42b7bafa","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.5 Flash-Lite · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b05f8827831fc092f12f06f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4baeec2b239cb5d078f29cd956977958a21794c23f6389f5d302e22740aad29c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · MiMo-V2.6-Pro · intelligenceIndex"},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mimo-v2-6-pro","upstream_model_label":"MiMo-V2.6-Pro"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"dc7e5fe1540bceaa27672f64e206471addf3239d02947c5bbda77ef24f261141","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.324207,"score_display":"46.3","source_stated_rank":null},"run_fingerprint":"3580374a12449ceb8038080ad251e6e226bcd6d98256b2c1d8dd55b5b180119c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Pro · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ad5c4858e74f428a0183b9f8","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a18c35fc4a8f7c8ff34671c7ca9ea01a4e6b09a97f4f4559fd241e776068b48f","metric_details":{"license":"MIT","variance":23.023675876359167},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1489.4801917138047,"confidence_low":1470.671204204633,"sample_count":4056,"score":1480.075697959219,"score_display":"1480","source_stated_rank":26},"run_fingerprint":"e77122b6fb009307e6e9c092d1a56f44f150b04a2c5656785fa213e0f7a73f49","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_95f69c97aa4a1cec75c671d7","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"399ef6689e1b0b812538f4a11b3e154eddaa428effc83dbeb757d9739c70641f","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · MiMo-V2.6-Pro · hle"},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mimo-v2-6-pro","upstream_model_label":"MiMo-V2.6-Pro"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ad8f4e1fe983dfc47308db3f1de437593df4b63ef51fa0f824ab617454e5413e","result":{"confidence_high":51.459953,"confidence_low":47.244855,"sample_count":2158,"score":49.351251,"score_display":"49.4","source_stated_rank":null},"run_fingerprint":"9909d9d7a01c3ef02046229bddd004b2cbb87354bd5e04ab5758d9fbecbdb379","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Pro · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ad5c4858e74f428a0183b9f8","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"97162a99c3f9b818b406f293fab7fe5c300a306aafad7533da520bc1cbf91138","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · MiMo-V2.6-Pro · critpt"},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mimo-v2-6-pro","upstream_model_label":"MiMo-V2.6-Pro"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"46ce9884ab657555c9bd69ee05f18a90f42940b889d1138dec2225d2801c859b","result":{"confidence_high":37.938769,"confidence_low":17.641826,"sample_count":70,"score":26.571429,"score_display":"26.6","source_stated_rank":null},"run_fingerprint":"e06f611ae1860047c059f027c854f13b907e1ed235d623792c99da06d4d6fd25","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Pro · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ad5c4858e74f428a0183b9f8","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"50424b981fca2a9e3fd8eefd269fb277b372a3354a56cb023962d2dab18c4b1f","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.835497,"date_is_proxy":true,"latency_seconds":8782.516,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.406},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":null,"source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.6-pro"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"09ab917cf288a5cf8a4533caeea13ff3c138c77556afaf71e5d12ff9777c0867","result":{"confidence_high":44.04876,"confidence_low":34.617239999999995,"sample_count":null,"score":39.333,"score_display":"39.3","source_stated_rank":32},"run_fingerprint":"2fe85b853a45a138d090c8d6fbaa79dbfd6759c5cb4e2a36f4eb7cc6053a8d70","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9371c32d8313eb1b572415d4","provider_config":{"source_id":"vals-ioi","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b9649088bfbcd770bcf96f1807baed07c32f7d2c41165b50bae3060d4ac4fe11","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · MiMo-V2.6-Pro · scicode"},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mimo-v2-6-pro","upstream_model_label":"MiMo-V2.6-Pro"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"9fe657cbcf686fa8e25b23ddb575c7643af71fa59785377e83dc65e2a09954a2","result":{"confidence_high":66.337372,"confidence_low":55.135465,"sample_count":288,"score":60.87963,"score_display":"60.9","source_stated_rank":null},"run_fingerprint":"a07fd794d596f0ae11018268a56680319cce78cde152f9c55ca22df7322f1d79","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Pro · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ad5c4858e74f428a0183b9f8","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ed178a3c97db87fff78d12bd741fbaf20a212f7bcf9a162d1be6de42298a34b2","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.043846,"date_is_proxy":true,"latency_seconds":3646.574,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.392},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.6-pro"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"279602edd827fa81b0e53353ded672cfa0c6459a7f356410b7a70dc443105d87","result":{"confidence_high":91.87132,"confidence_low":78.57468,"sample_count":null,"score":85.223,"score_display":"85.2","source_stated_rank":12},"run_fingerprint":"d951d34c27c7efbfac4ecae7add86a446ca09ceec40dad1ffec48752ef973114","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7c73c8831eb69ca88b4cc5e8","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"62c3de97de231148aa1b1cdf29bb837d930de174eac84d53133e12e316ca8305","metric_details":{"license":"MIT","variance":39.03002681021062},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"mimo-v2.6-pro","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"02623930d660369096b5cdcc813f8c8481d595a2d81c47611139504bdc3d7672","result":{"confidence_high":1630.658689652558,"confidence_low":1606.1693253682804,"sample_count":2734,"score":1618.4140075104192,"score_display":"1618","source_stated_rank":24},"run_fingerprint":"b7c39524ed9e5ba3b67be50e1ebbe5368894e87ab9e5b0325298e57e4e6ad18e","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_aeabb9ac695c1ca1e41af8b9","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d6b0deb9816e2f34267a9e3d7d1b7c54194af1abd1968cbf4346c33065387e46","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.072656,"date_is_proxy":true,"latency_seconds":939.123,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.873},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.6-pro"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"41c58025a16dc798d9687f4cad86556e28d9aeb59bc60922ab675be4e4ea6dee","result":{"confidence_high":71.46108000000001,"confidence_low":64.11892,"sample_count":null,"score":67.79,"score_display":"67.8","source_stated_rank":32},"run_fingerprint":"44a5c78ffdfeb09789c79120be1f7db9338dae078ac759bcb0730d7d6a810b6e","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_83ef76f94df553775bdb50c2","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-01T22:31:28Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T22:31:28Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"55914ef2ad8a9910846594cb37c30d405479c04db04126849ffa5a6065a241ce","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · MiMo-V2.6-Pro · gdpval"},"model":{"id":"xiaomi/mimo-v2-6-pro","name":"MiMo V2.6 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"mimo-v2-6-pro","upstream_model_label":"MiMo-V2.6-Pro"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"a7015f613ebcd15228e3bfec2bc513b85ea677c03b86b5b11691b5d9bc0a3f5c","result":{"confidence_high":1709.39,"confidence_low":1662.86,"sample_count":220,"score":1686.12,"score_display":"1686","source_stated_rank":null},"run_fingerprint":"4ed105e3e38951b6df357bbb0a5235d73746fdf3b7991ea56203f000e65e48af","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Pro · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ad5c4858e74f428a0183b9f8","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8518a028e161b24a9eb774b399d926fd3c3e27d7f6ed60389cd0ece07d84a122","metric_details":{"license":"MIT","variance":8.04197409436739},"model":{"id":"zai/glm-5-3-max","name":"GLM 5.3 Max","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1484.0567022628923,"confidence_low":1472.9404236912478,"sample_count":17857,"score":1478.49856297707,"score_display":"1478","source_stated_rank":27},"run_fingerprint":"9df2707265341d7777282fe5545bb6507e1ae65ff67ae8b872f6f8d015fc1cae","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bef56aee9a4cf41cbf313cc9","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d3e5a8bbf8eab71245412c0914f22990d266d6507c8b938e1da41853c1a6c2ac","metric_details":{"license":"MIT","variance":17.40068774283745},"model":{"id":"zai/glm-5-3-max","name":"GLM 5.3 Max","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"glm-5.3-max","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fada59b1b9eb639ccf178f75d844f171e41132d67790d2842c93a122b286bf57","result":{"confidence_high":1631.1272331032224,"confidence_low":1614.7755939907827,"sample_count":7658,"score":1622.9514135470026,"score_display":"1623","source_stated_rank":20},"run_fingerprint":"5ff4074c7f5f6e92c96aa091478163d740127f13c3f01d5f2cc8724986024b24","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3c518b832110015391372ee9","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1b8953c738e278107e0b759fbbf12d0d6f23c09abfb2abd2507316d705fec66a","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-03-31"},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen 3.6 Plus; model release: 2026-03-31","source_accessibility":"API access","source_model_name":"Qwen 3.6 Plus","source_model_release_date":"2026-03-31","source_model_versions":["qwen3.6-plus","qwen3.6-plus_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4b8450071b87fc38945b4987773296d56b5efd3e1a3e19e5e5149d0b959c4d54","result":{"confidence_high":149.31,"confidence_low":145.41,"sample_count":null,"score":147.63,"score_display":"147.63","source_stated_rank":61},"run_fingerprint":"f455d5dea4869a98dfd805d967748827f9299f8befb9cb7e5b35f58d7d10e089","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bb2a46687273d3b0b9692880","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ce21db24339d91197e2daf9fc1c9d9be634a429f911e79beb28d43c836240a5a","metric_details":{"license":"Proprietary","variance":4.464906990480831},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1447.5428790958567,"confidence_low":1439.2599433017892,"sample_count":48062,"score":1443.4014111988229,"score_display":"1443","source_stated_rank":88},"run_fingerprint":"2529c3ca93ae22d9e913dab94d7b2828d4b450eece11b583563cf18a6226a48a","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_53cc057b7b9f88727dc017ac","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1a9e37c9bc32da94c1d537e2207818d3102677c2ce91d8409df83a6cab19bda3","metric_details":{"category_scores":{"Agentic Coding":41.36366666666667,"Coding":78.1845,"Data Analysis":69.91166666666668,"IF":58.342,"Language":74.98933333333333,"Mathematics":83.72475,"Reasoning":75.827}},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.90613095238095,"score_display":"68.91","source_stated_rank":57},"run_fingerprint":"d022f6c942be50ad30bed1291716b73f41c361e8925c56ff8a0d5da86efdd5d1","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4729b1465639558d924d0a64","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:34:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:34:06.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8d910a90daeb66641216134c76e99c6991a6b2f8ef845a899552b2d720dbf04f","metric_details":{"best_score_across_scorers":"0.44144144144144143","model_release_date":"2026-03-31","stderr":1.5718302238794237},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4LWNorP8Fy8V8doxirNFcH","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4LWNorP8Fy8V8doxirNFcH.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4LWNorP8Fy8V8doxirNFcH.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"dae7bde98b5cba1f6aa19f37811b32f4fb6cb69223e0e46edb2d865d8fd12424","result":{"confidence_high":47.224931382947815,"confidence_low":41.06335690534047,"sample_count":1000,"score":44.14414414414414,"score_display":"44.1","source_stated_rank":40},"run_fingerprint":"8aa90f19b3901e2878c2e38eef0aef6abbc0162e0185ba54a3e4de52b1066f2f","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dd42b471745c5d39f5d66180","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c0272d36bcde46faafe8ee90a6d86cdd695e07e6c5814bbb349189d834a4d94d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.022964,"date_is_proxy":true,"latency_seconds":122.081,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.668},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":88.384,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.6-plus","zero_shot_accuracy":86.364},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"9af31c8ba0c89c293a9fc7c571ab061321ad14e1b4c81a985a7547ff0650d369","result":{"confidence_high":90.64327999999999,"confidence_low":84.10472,"sample_count":396,"score":87.374,"score_display":"87.37","source_stated_rank":38},"run_fingerprint":"80661ada6398b9c5677fdc8b03b4b189dfec84d25169f506cb78da7e5ef84499","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6fe23e846fb827f588fa93e4","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:31:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:31:39.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e433854e9d5d1be36d79fda7172d8e32bbb95719e7bb9116ff6b239b577d825e","metric_details":{"best_score_across_scorers":"0.8838383838383839","model_release_date":"2026-03-31","stderr":2.2828881775249354},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"f6ZdMtRpdfskGbrBCzrKSn","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Ff6ZdMtRpdfskGbrBCzrKSn.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/f6ZdMtRpdfskGbrBCzrKSn.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"6b5e5b17aa4a1d75cf0ab14c92501d6759368b330c6be1eb8bec058b98372994","result":{"confidence_high":92.85829921178725,"confidence_low":83.9093775558895,"sample_count":null,"score":88.38383838383838,"score_display":"88.4","source_stated_rank":58},"run_fingerprint":"fe5bd6bcbe8e491dce4b1fde1c14655b818384b77bd59ef17fcf265fd8db51cc","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e29854b177b8d9706164b4d8","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":178,"at":"2026-04-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-10","status":"stale"},"measurement_id":"eab6f2081a985c4f64968d6c4f3770b1283a9a4d345f93258d15e2b50128fc56","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.008813,"date_is_proxy":false,"latency_seconds":46.184,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/alibaba_qwen3.6-plus","source_locator":"Results · Overall accuracy","stderr":0.323},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":65536,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"efa5c81b5d51f5a4eca9807a0831d304c329af79d1b7b587ae6eb4aaa3bf7ca3","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.6-plus"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9095a399e4b5178a51d50853ef3afbe830254524c5757df4bec0ac7b700afe87","result":{"confidence_high":88.30108000000001,"confidence_low":87.03492,"sample_count":null,"score":87.668,"score_display":"87.7","source_stated_rank":24},"run_fingerprint":"74d78993e7aca89ab3816c67a435b7f594ffcc1fb48e57553d94b5a55de9bdcf","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d5fe971d7fb213c8c9957675","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:16:48.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:16:48.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"22cadba8b013f7ed78446d28819912ebc3c77fb520730de52f188ee99bc671d5","metric_details":{"best_score_across_scorers":"0.9333333333333333","model_release_date":"2026-03-31","stderr":3.760507165451774},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jYiGAFfUQKJuRStPzU3PX3","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjYiGAFfUQKJuRStPzU3PX3.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/jYiGAFfUQKJuRStPzU3PX3.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a92fa1db59b2ffe3eecd89b77e8dd6e5b3b50056ab0da8fc1a68ad306cc3c8cc","result":{"confidence_high":100.0,"confidence_low":85.96273928904785,"sample_count":45,"score":93.33333333333333,"score_display":"93.3","source_stated_rank":53},"run_fingerprint":"17ed0829832ce89b991a3bc573166329c1c5cd8034007790c8f0c239c96bd8fe","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_edf2f7cfb47d9da6a872a4c4","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:34:20.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:34:20.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"649b5bd7f996c9dd814a281d35d45ee4cf8a92eeb6f30c2297c7ff5a269b2e6f","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.3824561403508772","leaderboard_private_problem_count":285,"model_release_date":"2026-03-31","public_example_count":10,"stderr":2.8838027580159498},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["2026-04-02"],"epoch_id":"jbyVWnEvvFVNNc8AEF6WcQ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjbyVWnEvvFVNNc8AEF6WcQ.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/jbyVWnEvvFVNNc8AEF6WcQ.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen 3.6 Plus (2026-04-02)","thinking":null,"upstream_model_id":"qwen3.6-plus","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"fc60dfd114311ececfc1f57f9d9205b49465b7c6848b8e45b6aa556afb02d848","result":{"confidence_high":43.89786744079898,"confidence_low":32.593360629376456,"sample_count":285,"score":38.24561403508772,"score_display":"38.2","source_stated_rank":58},"run_fingerprint":"eb2a798b085adfd6c4d1eeb4354c115df74677c4b3d39352f3cfadc944503d53","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_776ab91b9e881c9d284f77fb","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-29T11:06:40.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-29T11:06:40.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a98f5a1f2eb4b77bd08ae0c0576d2c39d73b8a7b158c3b99dfb7a3971ec2d1e0","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.32280701754385965","leaderboard_private_problem_count":285,"model_release_date":"2026-03-31","public_example_count":10,"stderr":2.7743965653274283},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["2026-04-02"],"epoch_id":"Z4xfwTke5pR6RrY952bttA","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ4xfwTke5pR6RrY952bttA.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Z4xfwTke5pR6RrY952bttA.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen 3.6 Plus (2026-04-02)","thinking":null,"upstream_model_id":"qwen3.6-plus_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"2faae330d6fd754449eb50938cf82a446ae41069d7313891065d0f668241dbd6","result":{"confidence_high":37.71851902242773,"confidence_low":26.842884486344207,"sample_count":285,"score":32.280701754385966,"score_display":"32.3","source_stated_rank":68},"run_fingerprint":"f83dd7e98a5fd9af6abaa3489c377282b7dfee86681d9214c636f506280aded4","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_8bdcdd19b3ff7aa84074c7e6","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":178,"at":"2026-04-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":"2026-04-10","status":"stale"},"measurement_id":"1a6eb8da80a4e399b739223b16d5eeee056cb2ae03e3a20dd124c6000d8936c3","metric_details":{"benchmark_page_updated":"2026-09-01","cost_per_test_usd":0.017413,"date_is_proxy":false,"latency_seconds":111.943,"result_evidence_match":"dated-exact-score","result_evidence_url":"https://www.vals.ai/models/alibaba_qwen3.6-plus","source_locator":"Results · Overall accuracy","stderr":0.878},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65536,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"4d74f2738ff7b8e86e763dda0c8a032eccb06e67d28d3daace6503fb10c2bce0","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.6-plus"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1070d69c68224387c68c7569c004df5b0c334c4873d41c36beab67aa398d19b6","result":{"confidence_high":85.88288,"confidence_low":82.44112000000001,"sample_count":null,"score":84.162,"score_display":"84.2","source_stated_rank":27},"run_fingerprint":"4d9e82dd06ffa3dc2843a9065b62861f85e308c28e9f1eb4a9525b98226e7e50","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cebb40ff99c3605ea64ee86f","provider_config":{"source_id":"vals-mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e3b1e3693c1ab614ae7290df8da8f897f5022cfa3f74fbe17c9326dba25d36e3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-31","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.6 Plus","source_effort":null,"source_model_version":"qwen3.6-plus","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"e5a3c77a8b1382af9e48713c15079fa8b4cd1a9317f97f925927aa6a2fd6702c","result":{"confidence_high":9.760522,"confidence_low":0.793426,"sample_count":71,"score":2.85714285714286,"score_display":"2.9","source_stated_rank":127},"run_fingerprint":"0f3bddf3f0ab15332d020c4d4ff2c4e0b539a9a5f9d36dcb0aab5bcfb8817cf2","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5525cd61422623d479a15e5d","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:34:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:34:52.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3a7b86d0f8c8b2e7eaf3fa1b89ec1d9f01c08968961ccfd1ea810d7a407b016e","metric_details":{"best_score_across_scorers":"0.17","model_release_date":"2026-03-31","stderr":3.7752516806863685},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"RXsYuuSedkYZ5ubc8swxGp","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRXsYuuSedkYZ5ubc8swxGp.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/RXsYuuSedkYZ5ubc8swxGp.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a55b5f637451c823a7cac1e549373bc8f36ab46d537bfe66e7e39265fcbf7c05","result":{"confidence_high":24.39949329414528,"confidence_low":9.600506705854718,"sample_count":100,"score":17.0,"score_display":"17.0","source_stated_rank":104},"run_fingerprint":"4ecc4c6becd069bee380fad6b247e7f2e50fde7a60b7158580dbc9c9c8903232","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6d51b15144d16df42f8e1846","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7bf13dbcdcc5cd7f2cd0ad378881cf8145403d581b33e784f0c23f168ef6bcde","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.869211,"date_is_proxy":true,"latency_seconds":424.172,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.978},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.6-plus"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"a7cd11dad1bfe07deac5528563f0f658b7d82903f4b159b7ab5a879adadbc597","result":{"confidence_high":77.27688,"confidence_low":69.52312,"sample_count":500,"score":73.4,"score_display":"73.4","source_stated_rank":52},"run_fingerprint":"ce6a3eab9c136523c6b0de77d102dee1ffe989504a6bb2737c72dff9eca4f7e3","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f708ff27ef338fc68c717da0","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-14T17:28:26.954Z","basis":"evaluation_started_at","evaluated_at":"2026-05-14T17:28:26.954Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"0ab02e37ff9c72b9a9b755e92eaa483698de1c17d0a8ab94fa85faa6399e58e5","metric_details":{"best_score_across_scorers":"0.5785123966942148","model_release_date":"2026-03-31","stderr":2.246855652298021},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"8HV4HP9EM5KuLSwzifuLye","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8HV4HP9EM5KuLSwzifuLye.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/8HV4HP9EM5KuLSwzifuLye.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"c09ba031df9eea144b33747cbdd5528c2360912b4f94324407a7c5912c669de1","result":{"confidence_high":62.2550767479256,"confidence_low":53.44740259091736,"sample_count":484,"score":57.85123966942148,"score_display":"57.9","source_stated_rank":32},"run_fingerprint":"950da4223eb8ecafaa61b4f433e86edb2bba1e5b1aa56e813cd9c465f23a4dc0","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9af2e459ddfea2bce6eccd27","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a9e604a04d91ee0a500e226437d02b250cace037c0bb5dc7ef771f39c0f95e82","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.077628,"date_is_proxy":true,"latency_seconds":475.705,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.977},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":65536,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.6-plus"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"183664e545e12026b3baf4153a6700e3d6ee8d35def788333ade0a841de81839","result":{"confidence_high":87.86692,"confidence_low":84.03708,"sample_count":null,"score":85.952,"score_display":"86.0","source_stated_rank":25},"run_fingerprint":"e0d3a44c4290dbb17d319dbde811b45fdbf4d682a686de3f011b234a3892757e","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9a3feb733930bc036f1939eb","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f3fa3520d4b846a36086e5ee081afff1ef5998c5a897f3bd90d2330c490ab096","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-31","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.6 Plus","source_effort":null,"source_model_version":"qwen3.6-plus","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"51e091bd6dcb17b0442880d3b04b67d9a6262a3c333e620a64de850ae5139c31","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":40.7407407407407,"score_display":"40.7","source_stated_rank":143},"run_fingerprint":"6cfb6a8d2f4228b9c86423560e2489276eee889bce475c1c9c498c7f3ed255ae","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3af555d70a9697e425c2e5eb","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f7d6e0f144baa6b339737c33834cfb760d66e73508538d56f5e08e43cb3943bc","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.057069","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-31","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"qwen3.6-plus","source_effort":null,"source_model_version":"qwen3.6-plus","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"dc65d046db4bddc92135c312fd5d0df89094887ab9bdc53dbe4190c45af29f24","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":670.15,"score_display":"670.15","source_stated_rank":82},"run_fingerprint":"8361374eb089c304ff1d7cc832aede3fb3bef55f21747c3930bfad1814995c2d","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8201aa5a717e76400d712e5b","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"62611425741da56de1234e16342559eaa55af4a3476c84ac3e42f2b052d6a02c","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":5.447636,"date_is_proxy":true,"latency_seconds":2289.977,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.906},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.6-plus"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"17533ac4e86b475a72aeec03b64c3caff8c6c09cd172c46ada52b8793e51836a","result":{"confidence_high":33.22076,"confidence_low":17.90924,"sample_count":null,"score":25.565,"score_display":"25.6","source_stated_rank":69},"run_fingerprint":"2e61490d2fe47e45a7fb861abd0cd06dab572d8fdbef548bcdda5dae1ec66c8c","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54ca3d703000d3359b931ea7","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3a9781fd22fe5b54e4509a11fd0fe7f781782f407928c25b5a399c9f06e2bf86","metric_details":{"license":"Proprietary","variance":8.059265429692354},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.6-plus","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"22b63552e90cd623b41d8d9c211b7e4bfc9dd89850d81f53d2da75f98f55b9bd","result":{"confidence_high":1466.2993445563766,"confidence_low":1455.1711216482558,"sample_count":18688,"score":1460.7352331023162,"score_display":"1461","source_stated_rank":65},"run_fingerprint":"5006d97c8e9c1636562ba34688efce422e659952982cf3cd5e4076ab78405d1e","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7a85759823e37f67a2277eb6","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a4e92772c957067a69ca11be7a92fb2db5f67fef2151a8db400fac80caf934b0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-31","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.6-plus","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"07ddb05983790ba6040dc47117dc7ed6564f07d7e7c188ca60c21a54edbd8b81","result":{"confidence_high":1466.25,"confidence_low":1454.81,"sample_count":17562,"score":1460.53,"score_display":"1460.53","source_stated_rank":65},"run_fingerprint":"e107184b4bf4d02c5ffb0ab5410d34efb56fb53d96de02f626da058f7cd63e99","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f48f814179ad9e6823e572ef","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"51aec2c743b663180d17eddbb418f10b94b6cf046ad3e48a9e0dbb9dd4bc805f","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.201902,"date_is_proxy":true,"latency_seconds":778.109,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.633},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.6-plus"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"f3522758d1ee1ef6ad04b4a33cb5a502cd148bf7c56047324df97517e457b47e","result":{"confidence_high":56.384679999999996,"confidence_low":49.98332,"sample_count":null,"score":53.184,"score_display":"53.2","source_stated_rank":56},"run_fingerprint":"27905b9a6dc21f420fa5bc28cb9a3539c2b5f11e2a4469ebe195efa26d4cee95","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_08482f26f2f92123fd7df7cd","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"088e9b90ea27412e0bc0b50d2cf833a8007abe47d23547ecec250d1511a92100","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-31","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen 3.6 Plus","source_effort":null,"source_model_version":"qwen3.6-plus","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"5245d7d556f59316a3b62d871bfdd955706a9c52ece3c9601115f737cc29975e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5114.872000000001,"score_display":"5114.87","source_stated_rank":26},"run_fingerprint":"a5b891ae38b0af5efef729c0fce446ad213c80a3f45a5fae33a041bb2dacddba","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_953234b43d2b7322c944aebe","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ada2e7cb71ea7e3554459050165d29496e7ccef560075ce46617f1076d31118c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-04-02","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Qwen3.6 Plus · gdpval"},"model":{"id":"alibaba/qwen3-6-plus","name":"Qwen 3.6 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-6-plus","upstream_model_label":"Qwen3.6 Plus"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"46969a0950d1d3a0a0f296cfd9b432c2a526f755933db357fcd12542ce95a972","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":975.84,"score_display":"976","source_stated_rank":null},"run_fingerprint":"fe0965d0779a76a37870f36fdeea9b483d2b1bd7ca884025a66811328f068610","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 Plus · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_93750ce462b5c95291c9db8b","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6736ddcb252da6e8b76f23eb239addbda0e85e6c0154390fbc7d5b65a8837c07","metric_details":{"license":"Proprietary","variance":4.265186443422251},"model":{"id":"openai/gpt-5-2-chat-latest-20260210","name":"GPT 5.2 Chat Latest (2026-02-10)","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1479.7623811359551,"confidence_low":1471.6668173915357,"sample_count":35937,"score":1475.7145992637454,"score_display":"1476","source_stated_rank":33},"run_fingerprint":"370ab7d0cf22f58de90eecc96bdb6bc87e7de9687545ece93e4b91015dbb49de","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1c79998e7b3075ee4bedf479","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"54fa33e419eef81b25af9f9e430bcf285dc4a25006902de31f2a58ee036dc4fa","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro · intelligenceIndex"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro","upstream_model_label":"MiMo-V2.5-Pro"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"cd1b99fcf0758208f20f564e91de0cb6c48a81e76b61b48d89a51dfb6856ede6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":26.407814,"score_display":"26.4","source_stated_rank":null},"run_fingerprint":"0460c0ca0a3a90ef130f102085e0de0b0379a49fd3d1767e88ff2ad809d8e9ee","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8fe48d99b25cc95375221a5f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"20ab8fc4cfc4aa153d1a99d3fba0fb4caf8f84b3512fe9e523bfe18e5e7fc5d6","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro (Non-reasoning) · intelligenceIndex"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro-non-reasoning","upstream_model_label":"MiMo-V2.5-Pro (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"7365be2f0f0f611c661a01f596bbbde7114477808b5dffd30cf75a5995e7371f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.290033,"score_display":"18.3","source_stated_rank":null},"run_fingerprint":"b937aca54fc8e98cb0f3326e37a18723b8c52ee795727d4b021f7e308b70eb1e","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e913b39b0e5f0ce3bff13c80","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0f4bbd267647d770ba745b7344166b1989c02e7caba9530b55711c4f90c35492","metric_details":{"license":"MIT","variance":3.4115506999886307},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1471.1348675378533,"confidence_low":1463.8946102140515,"sample_count":70871,"score":1467.5147388759524,"score_display":"1468","source_stated_rank":49},"run_fingerprint":"ebc030d7f50fc8715a91cefa2922040e7e57db6af37b7ccb43fa392f27f0ee44","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cf88c4c74c23c65e8505ad74","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5f3023448c2ae4632fe1a43b","split_or_window":"public-multimodal-mixed-run-configs","unit":"percent","version":"reported-runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"evaluation_at","evaluated_at":"2026-04-22","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"decc6b5e09456be99db61bfc3196dac55313bed2c73ac3031a3cb7a74a069bb0","metric_details":{"comparison_warning":"Reported runs may differ in harness, tools and evaluated subset.","comparison_warning_code":"evals-report-mixed-protocols"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"evals-report-mixed-protocols","judge":null,"run_config":{"evidence_status":"verified","model_url":"https://evals.report/models/xiaomi-mimo-v2-5-pro","source_model":"MiMo-V2.5-Pro"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9bd3fcd30506d02bd2fd6283175199d72d7010226d22784e554ab0bbbf43b033","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.8,"score_display":"33.8","source_stated_rank":21},"run_fingerprint":"8bf66fe5ead5c0c61ca5a7b84a0901923b58ddc74248303b3751dc3b5f234d95","source":{"content_hash":"bbd790f846f0f01ab093f9bfb7fada054c2980fb08ac613d3610fdb071e69ea4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores","id":"evals-report-hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"CC-BY-4.0 as declared in dataset metadata; upstream licenses apply","locator":null,"name":"evals.report · Humanity's Last Exam","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://evals.report/benchmarks/humanitys-last-exam?tab=scores"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dd5b317ecd1d74199c314ed7","provider_config":{"source_id":"evals-report-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"evals-report-verified","verified_status":"evals-report-verified"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2ebf420eaed35240fae9c6ed9e62f7a1cd5657792796ff82f25eab23e075017a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":"with tools","num_parameters":1023182841600,"pull_request":1,"result_file_url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro/resolve/refs%2Fpr%2F1/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.5-Pro"}},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.0,"score_display":"48","source_stated_rank":18},"run_fingerprint":"a9343f9916d24b9f6732942af7535d9d09ae1608b516d4511f96ed0d5deed2da","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_08d1e210c1f4efbcc2c99f85","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"171f4dc9265fdbcaf5878363b95641df8e7fc6333c304a3d40f165fa38717c49","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro · hle"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro","upstream_model_label":"MiMo-V2.5-Pro"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"cd0461dadd329d0be451543ce165ca81ac524d86875590417582db8b5908c511","result":{"confidence_high":37.726239,"confidence_low":33.687022,"sample_count":2158,"score":35.681186,"score_display":"35.7","source_stated_rank":null},"run_fingerprint":"c0a8657a372f1d53876e15d69027514d3ac33939aa9f56bc91d774a7e7f09ee3","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8fe48d99b25cc95375221a5f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d09b3e8ddc72e391fb663a28e6527045415b9ee3e71980eca926f22314e64274","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro (Non-reasoning) · hle"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro-non-reasoning","upstream_model_label":"MiMo-V2.5-Pro (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ff5fe7e08a32a1a285a4b87b559bd047f949ee457e4e565425d8ead136531176","result":{"confidence_high":16.342259,"confidence_low":13.347317,"sample_count":2158,"score":14.782206,"score_display":"14.8","source_stated_rank":null},"run_fingerprint":"5b94efed708f010b97321761914ec59385065cce175a677419fc91eb9996c9d5","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e913b39b0e5f0ce3bff13c80","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1fdbf69c9fb0e0b77880cde181a0d5b48ea6a092233a9df27aeea362222990b5","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.006654,"date_is_proxy":true,"latency_seconds":251.332,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.468},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":80.808,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.5-pro","zero_shot_accuracy":84.343},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"15a7ac228cd7d32794eff7da7775b36c2a4b5b9e1a820cb596833b4223afa9f8","result":{"confidence_high":87.41327999999999,"confidence_low":77.73872,"sample_count":396,"score":82.576,"score_display":"82.58","source_stated_rank":61},"run_fingerprint":"84c6a50c290ac6c0cd8a163195c96f1e96f09d3d5d18cad8e95aaf31317f510b","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e8c081afa28a4a38e26e8d28","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"64930175c6015653bcdb57605e8f4660555ef4a06a698d59b7ab1c8c0b9e4302","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro · gpqa"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro","upstream_model_label":"MiMo-V2.5-Pro"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"ddb5772c56dc803a6af2bb871e5097373c332b14654ae38eb89b255c80991d50","result":{"confidence_high":90.625606,"confidence_low":81.113819,"sample_count":198,"score":86.565657,"score_display":"86.6","source_stated_rank":null},"run_fingerprint":"c1ed4e73dae73d0c2f27a5e4d43d2e6c22e741730fd894adc60f21204226397f","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8fe48d99b25cc95375221a5f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"51a82b67727779d1f6cb4d91a4f560cf5cf78ab4d8a8fed63b7f3eba020aac8c","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro (Non-reasoning) · gpqa"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro-non-reasoning","upstream_model_label":"MiMo-V2.5-Pro (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"e018dd4ea3253f9607401858b102f921a066eeb5f88d0cc2e73ef2880563249a","result":{"confidence_high":81.563113,"confidence_low":69.764264,"sample_count":198,"score":76.161616,"score_display":"76.2","source_stated_rank":null},"run_fingerprint":"650a89d1fdfc4ab7b37009bf06887e4ea536fe3b4efd810a40b96890c9f8fa83","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e913b39b0e5f0ce3bff13c80","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"359d7298829c32c24b968dad1a2c274f169853911f9b2aee3cfcfcf220eb5905","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.002085,"date_is_proxy":true,"latency_seconds":67.757,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.453},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.5-pro"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f8aa1906c9594b57c41c53730adbbb67bf6756b5b388dc633f8643229d510a2f","result":{"confidence_high":85.48288,"confidence_low":83.70712,"sample_count":null,"score":84.595,"score_display":"84.6","source_stated_rank":62},"run_fingerprint":"615cfcdf519af0de1b6fee9f908f7274e39d8305c44469db72772b35ad944f27","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e23668cebc6abd250154d2c3","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d0fadbf334461f72c8ddad36c9788cd238230606eea1df22b6d99de9fd32a838","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"","notes":null,"upstream_source_url":null},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"MiMo-V2.5-Pro","source_effort":null,"source_model_version":"mimo-v2.5-pro","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"faf6317b9f35ce799580466c00eb5a5baf6f06e8911eb403791d5d82409b25e4","result":{"confidence_high":11.389661,"confidence_low":1.332676,"sample_count":71,"score":4.0,"score_display":"4.0","source_stated_rank":114},"run_fingerprint":"0567d65f27d207cf0c324123b601975ec492404cc8fc7afb62356ff3a114a530","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d3dc85cd45955eb4c3bd11ca","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ad70188e8ccd2ed2e780523a84ec878426d6333cf8a0f6ed4871aca25d2f6c19","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro · critpt"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro","upstream_model_label":"MiMo-V2.5-Pro"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"0c3ea6fcbb859d82df8441105ff8a6e25c282fc65d419aae0af8d4f2a9d33f39","result":{"confidence_high":11.463124,"confidence_low":1.323165,"sample_count":70,"score":4.0,"score_display":"4.0","source_stated_rank":null},"run_fingerprint":"cbc911b2c0a893534395eba203ed18d6876599c1f7dfd997109aa7f8efd2acb1","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8fe48d99b25cc95375221a5f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c7a6d12fa1d7cd5211bc80cf67ee3db5ae42756fffb757ddea8f1fe4e6e435a1","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro (Non-reasoning) · critpt"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro-non-reasoning","upstream_model_label":"MiMo-V2.5-Pro (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"fce9f6fd93f9b8e0de986b37f4022ee742e4c818ae3c1cf808d198326ea110c3","result":{"confidence_high":7.197254,"confidence_low":0.172034,"sample_count":70,"score":1.142857,"score_display":"1.1","source_stated_rank":null},"run_fingerprint":"e0b790b1d651031e49079723e37f85ebd123ca5830c97ab96602119f75fbe71b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e913b39b0e5f0ce3bff13c80","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d86ef438941f7874066e71aa28e40bd44623b09811d9a467e15cd4dd047c2345","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.022968,"date_is_proxy":true,"latency_seconds":470.814,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.964},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.5-pro"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"b28ece5b3f995c55ef14f472f5ec5cc8ffa774525e6f9324c6622c7ec0a47c97","result":{"confidence_high":77.84944,"confidence_low":70.15056,"sample_count":500,"score":74.0,"score_display":"74.0","source_stated_rank":50},"run_fingerprint":"5296eda5fb8a2b42470d8b5f8fa40a5a73a606e14161c519208c1dfd0f58ad50","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0785c90f25f84cb66ec81391","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"9c37c659f201b32987eff894cfee9fb41b5a9c32ae19a45c2474faab2dffa471","metric_details":{"model_release_date":"2026-03-22","pass_at_5":65.76576576576578,"potential_contamination":false,"sem":0.5405405405405403},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"MiMo V2.5 Pro__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"49ad2dfb26d0e8051efe67dd32de507e6b83d1a5fb53f1b6ece7b758fbea33c1","result":{"confidence_high":47.54594594594595,"confidence_low":45.42702702702703,"sample_count":null,"score":46.48648648648649,"score_display":"46.49","source_stated_rank":8},"run_fingerprint":"cd7e714afccb62b9c0e75bca9a25f10c60e2f378e0f231976a7859a55918de3d","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1fb89a7ed9a95fce8391c709","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"15a730cd421458c57e41ed6503c784b36899b19d9f5db00809e3450493b5bb9d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.006808,"date_is_proxy":true,"latency_seconds":271.074,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.069},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.5-pro"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"660fade519e035d0a2510b1600bc476be3dce68e91384bcf800ac77e55253f8c","result":{"confidence_high":83.44924,"confidence_low":79.25876,"sample_count":null,"score":81.354,"score_display":"81.4","source_stated_rank":64},"run_fingerprint":"ef198f3d2a62b987ca94daaa97d6ae1f0b245bf0c985dfd78959864d248702c7","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_58d95fa53ab7d749ad91dc8b","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6370ad52754306895e337ba57979747b7e1529aed6a9259c2531cd9e60120965","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"MiMo-V2.5-Pro","source_effort":null,"source_model_version":"mimo-v2.5-pro","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"e77e371acf4336f05f77ff67729676f9e7924a5ae6d4584134e07e7d70528dec","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.2314814814815,"score_display":"50.2","source_stated_rank":87},"run_fingerprint":"df8e63658938e199872441dcc7788f5f083efcb0f15ee9c66798bb6eb8dccc34","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acb46f5b3108016822f0c8e0","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3969ea0f9457fe3c1435c535e6f3821d907d609c0bae7b46c337a774deef86e4","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro · scicode"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro","upstream_model_label":"MiMo-V2.5-Pro"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"16550fc7812aa94ec6a23cb6ff7c7e297fb824bf7eca5c4fb0e03d7ba4ff23f0","result":{"confidence_high":56.30728,"confidence_low":44.834893,"sample_count":288,"score":50.578704,"score_display":"50.6","source_stated_rank":null},"run_fingerprint":"5ac9c398c163269c97244afbfeb136b9d7d48ae6be7dbff7d11a473ed6e7524c","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8fe48d99b25cc95375221a5f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a0fcb777b8dedef48eccbc1f911d2fda034264e84b4793df7ffc4016eb78e68b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.102977","epoch_dataset_updated":"2026-10-05","model_release_date":"","notes":null,"upstream_source_url":null},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"mimo-v2.5-pro","source_effort":null,"source_model_version":"mimo-v2.5-pro","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"332e5148a14ee8714a7566d28a9eb053d3e7fa4e32accba9de0c5dac596f24c9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":899.8,"score_display":"899.80","source_stated_rank":59},"run_fingerprint":"63d235c44e9b40cb1aca694221d4341ad4b2d53e9cb5208cc65d978a3f67ffa9","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f27c99ca4ea89e5927d9d756","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9a9af75a526a7858c7e2947e57e395dee12fe667565d5eca4c6fcf219e5c6bcc","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.166747,"date_is_proxy":true,"latency_seconds":2403.764,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.529},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.5-pro"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"bce5703347c514119f8e2ae6b8a692777b53bdc24f01e4dbf0439b96bb837c8b","result":{"confidence_high":42.98984,"confidence_low":25.236159999999998,"sample_count":null,"score":34.113,"score_display":"34.1","source_stated_rank":64},"run_fingerprint":"fc335986332164fc8e4456f5fd728e9dbd8f82019bc78162af65e20c44e2afaa","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_33eda8b12928b09184788407","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5d9dd07f203c2d0fc5bd163910182c77e38df19aef99b005fee4df4b35b81e7d","metric_details":{"license":"MIT","variance":7.552947981406427},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"mimo-v2.5-pro","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"87b64e5cb104e9c157ca48dc04948e8c7273ee503543d928bef0b1775558f77e","result":{"confidence_high":1483.0312388654165,"confidence_low":1472.2582469605804,"sample_count":20313,"score":1477.6447429129985,"score_display":"1478","source_stated_rank":60},"run_fingerprint":"0c58026b8a8944600c03572112297d30197b02d04d18b6df5fbb5e4176ab7892","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acf227e528e7a25cffba9cf6","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2a050fd42f02b0049e609e0b2812f5b53c9d46f64dea5bea6d6549c82d3e9564","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"mimo-v2.5-pro","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"49eb7f07550537a121b70838af3e839cb72d9235949a9c104dca0afe50ed272a","result":{"confidence_high":1480.85,"confidence_low":1469.0,"sample_count":15843,"score":1474.92,"score_display":"1474.92","source_stated_rank":59},"run_fingerprint":"aa963d8c02e261a8dc5d09692ddaac40cea0f9be3e1b97426239dc8c698f96fc","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4bd25075ce03de8a82ecfb8a","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9faed9dbb88b6a4d87a90586018f302dd00c954b34c1de25e3a2c6449f09d26f","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.043613,"date_is_proxy":true,"latency_seconds":866.253,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.273},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.5-pro"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"f24f218d2406c26a4c73b480bc8be4c48a1a17112e4908b1b7764ae5cf0fa6c0","result":{"confidence_high":67.63808,"confidence_low":46.96792,"sample_count":null,"score":57.303,"score_display":"57.3","source_stated_rank":44},"run_fingerprint":"b48a99d8872e403eb485d3ab09b0a5dcb7e8c04877b794d37ab7c78e90c603fe","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6690d5e0f6e56e1612dfa415","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"97ea1e92766dd1a3a8ee79f5a0a78b04f761cba5f0d3ef9d2a31319a525db54b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro · terminalbenchV21"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"mimo-v2-5-pro","upstream_model_label":"MiMo-V2.5-Pro"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"e576444fb87670f37c9378fe66db12e63a4fcf7c6e176a390bf2c69d6b0c7bdc","result":{"confidence_high":74.252667,"confidence_low":54.829124,"sample_count":89,"score":65.168539,"score_display":"65.2","source_stated_rank":null},"run_fingerprint":"964c6f40ab5acaf71828f22276c13d8a6902252388c396382224b4253d399d1f","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8fe48d99b25cc95375221a5f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"19f83053193d8e48953be9fed7afd8f56318d8860a7b5ca079fa94f57f5332eb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · MiMo-V2.5-Pro · tauBanking"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"mimo-v2-5-pro","upstream_model_label":"MiMo-V2.5-Pro"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"66241d651244d29c262a1ee61a7e597df71b9eb9e7f7dae6ccb808ae74e61581","result":{"confidence_high":17.450034,"confidence_low":5.399266,"sample_count":97,"score":9.896907,"score_display":"9.9","source_stated_rank":null},"run_fingerprint":"ed630d4f7504cfb817a5183556f0a89e03a2679b5940beedae0d091a7b0fff20","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8fe48d99b25cc95375221a5f","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"175e8de430da2265028e77e41078a610485399c8bca7d7f5eb79d9a02f3e45e2","metric_details":{"confidence_level":0.95,"license":"MIT","session_count":45239},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Mimo V2.5 Pro","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9b7aa59f7286af4284195b09d9a83e69dbb6210a3cdb3040b6a1ee32ce2ec001","result":{"confidence_high":-0.06631839298385228,"confidence_low":-0.08410485818840951,"sample_count":2309389,"score":-0.0752116255861309,"score_display":"-0.0752","source_stated_rank":44},"run_fingerprint":"64ccc234d0520a6314099437b21fbbd483121b34f9e02d1be8db771429531fea","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d8ac0691b0cd4bec10937bd4","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"71f9527bc6ba5566feddc6a381a447f8d909c3680f356f55b2370aeee438bee1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · MiMo-V2.5-Pro (Reasoning) · gdpval"},"model":{"id":"xiaomi/mimo-v2-5-pro","name":"MiMo V2.5 Pro","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"mimo-v2-5-pro","upstream_model_label":"MiMo-V2.5-Pro (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"0b0d35873a1efc2a102a1bec80b2eee8612cd341ac9377050eea364d30f8f278","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1107.02,"score_display":"1107","source_stated_rank":null},"run_fingerprint":"410381addf2c3808ce7c1d31faa6656cb84ce11dca9ace023302028e7af775e1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.5-Pro (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_53135ec52b36e5ccd955ed0c","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1c892e765d1d7aef26425269a92d24c3ae8709a13f222a3e575bfad50af02070","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-05-05"},"model":{"id":"openai/gpt-5-5-instant","name":"GPT 5.5 Instant","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":false,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.5 Instant; model release: 2026-05-05","source_accessibility":"API access","source_model_name":"GPT-5.5 Instant","source_model_release_date":"2026-05-05","source_model_versions":["gpt-5.5-instant"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"830629b375704cfb37ebcb2ffd9051cd51416b6a5c746af24d090656ba7c601e","result":{"confidence_high":145.3,"confidence_low":140.03,"sample_count":null,"score":142.49,"score_display":"142.49","source_stated_rank":98},"run_fingerprint":"6bbfcf2691749b77caea2bc55120dad51ca31560a49a27fb73c65bc6fce3e5a4","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_68f21033bf83c6f8fbdc9a46","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d61b5dea2e29069e13225d3236dc2b68ac84f5b25817725c62bec093b7bdc1fd","metric_details":{"license":"Proprietary","variance":6.534480118896229},"model":{"id":"openai/gpt-5-5-instant","name":"GPT 5.5 Instant","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1478.2702142254457,"confidence_low":1468.2498477069225,"sample_count":27098,"score":1473.260030966184,"score_display":"1473","source_stated_rank":40},"run_fingerprint":"11045871f0a33e2ca0b7ee210245b2ad9878674451d2c0012bed6b2bd5896286","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_330cfe6b7fe94f55d14565b7","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":64,"at":"2026-08-02T02:46:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T02:46:37.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6dd7c25c2b41876f16e6d4013ff29e2adf6454667cd0a99fe744647b53ff5548","metric_details":{"best_score_across_scorers":"0.8251262626262627","model_release_date":"2026-05-05","stderr":2.2532782662252124},"model":{"id":"openai/gpt-5-5-instant","name":"GPT 5.5 Instant","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"WPVhgi8NT9grAT7kfipT9G","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"86d538da57fca9d7c91572c668d7e41e63f4c09d12c1080a6443217a5173c20c","result":{"confidence_high":86.92905166442769,"confidence_low":78.09620086082485,"sample_count":null,"score":82.51262626262627,"score_display":"82.5","source_stated_rank":113},"run_fingerprint":"422438e95705797caa4e8e8c6c35fbce54668bacfc7674dd3dfb36638c1d7265","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e642c9d37f1af0ef22ec3a14","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T02:46:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T02:46:37.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"574529c1a220ca1a4e2c73ab94d01abd79beae575592e94cece52da4d4322046","metric_details":{"best_score_across_scorers":"0.6805555555555556","model_release_date":"2026-05-05","stderr":6.212711436196764},"model":{"id":"openai/gpt-5-5-instant","name":"GPT 5.5 Instant","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hGzLQWbxPdgQiiMbaDqF8k","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"bd58235a562a986d5655b9fa3d5e4e24e5185b5caa9dee91090a4bb9727082a1","result":{"confidence_high":80.23246997050121,"confidence_low":55.8786411406099,"sample_count":45,"score":68.05555555555556,"score_display":"68.1","source_stated_rank":146},"run_fingerprint":"8ae27c9b4e3a9f2a50c2c5808aa19b0ba6ce0a89c667d352ce39e3c613ff5f20","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_13b94eb3aaf17b8576fe19ef","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T03:23:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T03:23:36.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e5807529904d673c704e474311376126810f01c4bbaf05ca98e9db34143aad1a","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.2631578947368421","leaderboard_private_problem_count":285,"model_release_date":"2026-05-05","public_example_count":10,"stderr":2.6129809832490087},"model":{"id":"openai/gpt-5-5-instant","name":"GPT 5.5 Instant","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"gGcHjE6ppy54ahN2pfxsGH","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgGcHjE6ppy54ahN2pfxsGH.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/gGcHjE6ppy54ahN2pfxsGH.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.5 Instant","thinking":null,"upstream_model_id":"gpt-5.5-instant","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"d88f5a819f66f4d19b051f0d673bb006c3c4fb4295841672338401037a42a31d","result":{"confidence_high":31.437232200852264,"confidence_low":21.194346746516153,"sample_count":285,"score":26.31578947368421,"score_display":"26.3","source_stated_rank":74},"run_fingerprint":"4731ea656424058350e16202adcd8d5149bc9a302da806e0f04ac3bd0726b280","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bd5c6fce5440443fd1b5a873","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T03:23:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T03:23:36.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bb915f57ed59cf5210d7572d83794b03263d8b93a6ddb062d577030cac85663f","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.024390243902439025","leaderboard_private_problem_count":41,"model_release_date":"2026-05-05","public_example_count":2,"stderr":2.4390243902439024},"model":{"id":"openai/gpt-5-5-instant","name":"GPT 5.5 Instant","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"2UmuBfhBvYbbnTg5BnQqov","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2UmuBfhBvYbbnTg5BnQqov.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2UmuBfhBvYbbnTg5BnQqov.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.5 Instant","thinking":null,"upstream_model_id":"gpt-5.5-instant","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"3f9de5475de6ffee0c5290dd7ce2901ff8918a8cf5990eb39af29fc63d5fd6e3","result":{"confidence_high":7.2195121951219505,"confidence_low":0.0,"sample_count":41,"score":2.4390243902439024,"score_display":"2.4","source_stated_rank":61},"run_fingerprint":"35acd02465ee52909ef7f889989a48688bcb9fb25c4c0fe36c9d188e42c4e615","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4e913c37f9e4dc8418e49c97","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"489108d2c9e1b70ede35462217f945f8d05532013d1120a763a9b7cf1aa99715","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-05","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-5-instant","name":"GPT 5.5 Instant","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Instant (June 2026)","source_effort":null,"source_model_version":"gpt-5.5-instant","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"90739739f322573cf0934f54aa88a344a4e4bc9723803153be1bae1ff77537e6","result":{"confidence_high":5.132974,"confidence_low":0.0,"sample_count":71,"score":0.0,"score_display":"0.0","source_stated_rank":167},"run_fingerprint":"2dd1a485be74f902072f08dbb4f6d8b3e2aff9323f770ee0ae97e5ad449e2c1e","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_002ac6ea591ffd737ee3511b","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":64,"at":"2026-08-02T02:46:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-02T02:46:37.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"38aa971fe7051299a9dbb7f013849c53beaec1d2430225f4453242502207a6dc","metric_details":{"best_score_across_scorers":"0.12","model_release_date":"2026-05-05","stderr":3.2659863237109037},"model":{"id":"openai/gpt-5-5-instant","name":"GPT 5.5 Instant","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dodABAysqTvxvThpS84BpK","log_viewer":"","logs":""},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7550bf6e80018fb7f893604218c777e6ffd47c932fc793a891f9ab2968ca7c7c","result":{"confidence_high":18.40133319447337,"confidence_low":5.598666805526629,"sample_count":100,"score":12.0,"score_display":"12.0","source_stated_rank":127},"run_fingerprint":"a32cea58c996d61b74dc476f1036edba3329c7fa1d3962479ed42a407dad3ced","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a4870a24a531b0893891ae34","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c39d8071ada0b659213d7ae21a873f2feefe2afc60658282f334f090fa8a4b1f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-05-05","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-5-instant","name":"GPT 5.5 Instant","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.5 Instant (June 2026)","source_effort":null,"source_model_version":"gpt-5.5-instant","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"d7550c39386d0afd190a112feab5a43e14e1e141f83a7ea3a2ce06073420e723","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.6111111111111,"score_display":"48.6","source_stated_rank":102},"run_fingerprint":"74b45a7aa4cb74475af31359a09bb3024a800cc22f61e15fd243fa5fa9a3146a","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0e35a90242db3ebf58471e83","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b7eb59c73fea51462988fed32416e225c6ff1cd1b79f54e8ac6684189bbc100e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-06","source_locator":"Leaderboard models table · Hy3 · intelligenceIndex"},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"hy3","upstream_model_label":"Hy3"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"3d23b71a40e2a9e28691c7c70a96116d44fec538dcae553f165aed46b9088364","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.297278,"score_display":"25.3","source_stated_rank":null},"run_fingerprint":"3684496f386c1d7afa223a2e3799ad0061cd55a58b7c96b4c721a7fc26bb3ebc","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Hy3 · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c679e74ea315691491c34a6","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"61c92e5b308b2e6220b022e772e5f2d80d6867a476e572732cd8d4cb19c478fb","metric_details":{"license":"Apache 2.0","variance":11.065733101107853},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1462.5033075751576,"confidence_low":1449.463590089154,"sample_count":10476,"score":1455.9834488321558,"score_display":"1456","source_stated_rank":67},"run_fingerprint":"81a05e182fbb532cf8a937cc1a1230ba805d42e46bd06fbf8866146a31907e42","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a509997ff81cf41e490ff500","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9d7dd20f4652fa0fd50431580d1451f15805430437133a650cad97ee7ceb727c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":null,"num_parameters":298786155776,"pull_request":8,"result_file_url":"https://huggingface.co/tencent/Hy3/resolve/refs%2Fpr%2F8/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/tencent/Hy3"}},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.2,"score_display":"53.2","source_stated_rank":9},"run_fingerprint":"ec3c484bde179f5a9f71caea4e09a56cb8aed52beaa3f9f0e6dfae8d53211e38","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f4211279c9136484fa19f66c","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4b69560e2ead79194c0fbc564856e3f041ce43449b9adf730dcf2cd9d21cce31","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-06","source_locator":"Leaderboard models table · Hy3 · hle"},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"hy3","upstream_model_label":"Hy3"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"707bc3a336a2b861823680295730e22b64c81c46e67bddd0033211baa7a47485","result":{"confidence_high":35.475536,"confidence_low":31.497068,"sample_count":2158,"score":33.456905,"score_display":"33.5","source_stated_rank":null},"run_fingerprint":"af16a52df532d6733caeaee50bb3da7c67a3face02608d4568b77d2c71247f32","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Hy3 · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c679e74ea315691491c34a6","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8ca3f175189ec737db3b377b26f258b76394cdbb2863a1a3d98478f6107c2f61","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-06","source_locator":"Leaderboard models table · Hy3 · gpqa"},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"hy3","upstream_model_label":"Hy3"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8c1e37104b29278e822dc59a31cd9454bc78399bcd462c4610555849fc3f48a8","result":{"confidence_high":93.202879,"confidence_low":84.679976,"sample_count":198,"score":89.69697,"score_display":"89.7","source_stated_rank":null},"run_fingerprint":"51fdfdfcb2e0178e14fbe9cb2f1fc474b8699445e0214fcf7128dfc338594faa","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Hy3 · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c679e74ea315691491c34a6","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"31a0db7db2aeb254c9117bf4df42cb0516bd36769cb434157e26a497bfdfafbc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-06","source_locator":"Leaderboard models table · Hy3 · critpt"},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"hy3","upstream_model_label":"Hy3"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"f1e7b70d358cff946876a8e78bb6b86de839635ff2813b66f3c7ac2762070ee7","result":{"confidence_high":12.642381,"confidence_low":1.769008,"sample_count":70,"score":4.857143,"score_display":"4.9","source_stated_rank":null},"run_fingerprint":"f8dbae224576260718e79e4eb023d2ecdac04c6731bc2f98336dbb7a3908bb4a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Hy3 · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c679e74ea315691491c34a6","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-06T00:23:09Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T00:23:09Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"35e82113ce96243cae97e6978d10b3f06aa680c8075c469950744c5cb3eacf25","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-06","source_locator":"Leaderboard models table · Hy3 · scicode"},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"hy3","upstream_model_label":"Hy3"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ccf4ea796ff65b94c58a3bd3bcd035f5fb7acf796c79e11290643e787276cc40","result":{"confidence_high":54.363781,"confidence_low":42.895005,"sample_count":288,"score":48.611111,"score_display":"48.6","source_stated_rank":null},"run_fingerprint":"93ea0b2a4cb0e2f6293688a742c9d009b49d4a6b879f5e6747c8cd424c305965","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Hy3 · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c679e74ea315691491c34a6","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"63749d8d2a220665ea524fda262c8e79dfbddd0a1ede5481cdf82ec7d3b7287d","metric_details":{"license":"Apache 2.0","variance":19.53634564549777},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"hy3","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9e27f1b2806d0fd2d04c78261f2e191acd3686463717b1f00a9b14876204f5e7","result":{"confidence_high":1516.1108505719988,"confidence_low":1498.78479305114,"sample_count":5802,"score":1507.4478218115694,"score_display":"1507","source_stated_rank":53},"run_fingerprint":"0e44cf2558723b1f5ce0fa3ba3e4d8ddf331ed85530a76ac1e5dce9a617bec4c","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_debe9c2cc9379011d1013ab2","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4976729cdd208e0a05a3019cb21134ea36465890d04f9be3761b90f014fb3ff4","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-06","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Hy3 · terminalbenchV21"},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"hy3","upstream_model_label":"Hy3"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"419ad250d5d913a5ca3a873a6279f52fcb1cda68adc33befa206e4bc65b9a522","result":{"confidence_high":73.579754,"confidence_low":54.0659,"sample_count":89,"score":64.419476,"score_display":"64.4","source_stated_rank":null},"run_fingerprint":"292eeca8fabd547f8822a02368fea430650eebb5f58c721fbf427bcc498308de","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Hy3 · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c679e74ea315691491c34a6","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"da559cbef2fa9b351b1411778a682b51017b5819f5ef389f88d59b42e1b454f3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-06","source_locator":"Leaderboard models table · Hy3 · tauBanking"},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"hy3","upstream_model_label":"Hy3"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"ab2e668e537bd41c7e1214b3473db5f0a222aa6e77caa3ed2ffb1e2700ac91ff","result":{"confidence_high":32.183877,"confidence_low":15.65511,"sample_count":97,"score":22.886598,"score_display":"22.9","source_stated_rank":null},"run_fingerprint":"1b091a33f5b172dd7c6d52008f5b0248261a4bde145bf3033acbfc56ea9992d1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Hy3 · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c679e74ea315691491c34a6","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1b1fccbcbe5c24ad4b8d50e4a6334e1d0e1e198969e0ac312c8705384f68336e","metric_details":{"confidence_level":0.95,"license":"Apache 2.0","session_count":29677},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Hy3","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e3bf69dbbb95d4e71fc50a012363b5c5d645cc9f4cac2fdc6cc762784d3df00e","result":{"confidence_high":-0.041674682291783414,"confidence_low":-0.06591497360554492,"sample_count":1171568,"score":-0.05379482794866417,"score_display":"-0.0538","source_stated_rank":41},"run_fingerprint":"78fb4e42dc359ca435cf67bf5fd40f6737ab94f1e73e14da919ceb19a9c70ba8","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_98914b89929791d91bf8e5de","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b1640c02d6b32aa87685afe4103ae6999511bef18014c975db61e8315a027858","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-07-06","reported_confidence_interval":"-27 / +27","source_locator":"Leaderboard models table · Hy3 · gdpval"},"model":{"id":"tencent/hy3","name":"HY3","provider":"Tencent","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"hy3","upstream_model_label":"Hy3"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"b5a53c8e0e32e7312b170d5d513dd7cf6c669c708b6e16e7101beb2de0029bab","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1045.12,"score_display":"1045","source_stated_rank":null},"run_fingerprint":"5588ea17f40f42caf40ee82f6eb07e6d396ef60d44f5287f0381c38905a10188","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Hy3 · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c679e74ea315691491c34a6","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cf9f60f354d2dca3870b6937ebf5d6b158109de76ce80d52f6f344047132cb2c","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-06-02"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen3.7-Plus; model release: 2026-06-02","source_accessibility":"API access","source_model_name":"Qwen3.7-Plus","source_model_release_date":"2026-06-02","source_model_versions":["qwen3.7-plus","qwen3.7-plus_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"01419811d8b982437335f0ec92e682dc8227f02f9e7c847a30cdb351a8e42724","result":{"confidence_high":148.94,"confidence_low":145.72,"sample_count":null,"score":147.38,"score_display":"147.38","source_stated_rank":63},"run_fingerprint":"3f399ea21ccc378308ede9cbc74916e094247dd317e8357a124d9b4f73490807","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e1a173ba85474b81dd96d886","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b270238edc49f101f0aef21c28c15b0aa6e75ebd0dcad3484c99fb896405ebe7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · Qwen3.7 Plus · intelligenceIndex"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-7-plus","upstream_model_label":"Qwen3.7 Plus"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"ade0f5a2b45c6236ebd819549a7ac2de5e72e7630614cafafcb96a848597ceb3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.162222,"score_display":"25.2","source_stated_rank":null},"run_fingerprint":"a34f9e55b359176e60e405550187543aa13c36e9542a923a5be4dcf6805bdfda","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Plus · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb3bf508162eaffa0f3e019","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d5a9a91114ffa249ba82825b4189173280d21ac75d18c9ad541986786c52efd5","metric_details":{"license":"Proprietary","variance":4.919163920792106},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1460.1219248235875,"confidence_low":1451.4278427825743,"sample_count":41908,"score":1455.774883803081,"score_display":"1456","source_stated_rank":68},"run_fingerprint":"fac9322bc7f22318ecfcdcd3a2611c67e1fcd131bd2be0af7d0b1275947bba02","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d371d7d5e0e9099eb888b168","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5b5563cd48bd10c552a97181","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"HLE · Qwen comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"960bd119403589c91a595638a9e0e2598df9a1517ac1fd38d926899b2043b92b","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Text Performance table; General; HLE; Qwen3.7-Plus column"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on HLE judged by GPT-4o; task count, split, tools, reasoning effort and run count not disclosed","question_count":2500,"reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"not reported"},"run_count":null,"scaffold":"not reported","tools_allowed":"not reported"},"protocol_fingerprint":"47241df9da3a012c78b147fc8290f4a7b8b00a7dbd528fd5149c65419be8495e","result":{"confidence_high":36.588172,"confidence_low":32.858777,"sample_count":2500,"score":34.7,"score_display":"34.7","source_stated_rank":null},"run_fingerprint":"dec78fb32c956d9452986860f5f511acdac52f778060d512cf40081c0362e35a","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Text Performance table; General; HLE; Qwen3.7-Plus column","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e292d3c46b34a34fb650826a","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c82ed1d7016a08e14384047ff43cd92f5cd672aed982067688f8c082da268d0b","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · Qwen3.7 Plus · hle"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-7-plus","upstream_model_label":"Qwen3.7 Plus"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a1c1d0387bc4df99e8c62ce8ff8151461e9484bbaabfc47c50742e734c81f818","result":{"confidence_high":37.679399,"confidence_low":33.641349,"sample_count":2158,"score":35.634847,"score_display":"35.6","source_stated_rank":null},"run_fingerprint":"3a66267d7ed2712be29ccceeb29fd6532846a2d93afe3524ea43097967bf4b73","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Plus · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb3bf508162eaffa0f3e019","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"492695d8d9d8aeacbfe1b98992281583d67a7b3a49239abe941ad44aafe5789e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · Qwen3.7 Plus · gpqa"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-7-plus","upstream_model_label":"Qwen3.7 Plus"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0dc6bc5d87e7425be332108b9c2fecedcc6aeadbfc64257f421a760daa2dd379","result":{"confidence_high":93.446904,"confidence_low":85.030476,"sample_count":198,"score":90.0,"score_display":"90.0","source_stated_rank":null},"run_fingerprint":"f350d6ee8f2f77207b073c3a60330b13785f9d06cc76339bb7b4a5176be3bf37","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Plus · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb3bf508162eaffa0f3e019","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:07:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:07:25.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"32d88ac5ce4d5ebc8ba805aff5eb8a0e10d93414ce8d38d2f7489ad6cecaa8d5","metric_details":{"best_score_across_scorers":"0.8787878787878788","model_release_date":"2026-06-02","stderr":2.325315795194205},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"URoWjw9Htc6QPwABVcrybw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FURoWjw9Htc6QPwABVcrybw.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/URoWjw9Htc6QPwABVcrybw.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"f526a57d83346107dc4bd9edd30ed7034330f9a8a860cfbcc8e0ed8a897810d8","result":{"confidence_high":92.43640683736852,"confidence_low":83.32116892020723,"sample_count":null,"score":87.87878787878788,"score_display":"87.9","source_stated_rank":63},"run_fingerprint":"7adf5f58dee39359874840ed0ce8c1d57f036df46a2d4cf86a0a8fd3c4fd1e26","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ff8f8f6ec3888292a9052577","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:03:42.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:03:42.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5105e6cbb651c4f6276939ec16fd5c70508b6fa9b517b8cb544d25df4c59ea9c","metric_details":{"best_score_across_scorers":"0.8181818181818182","model_release_date":"2026-06-02","stderr":2.7479603010538827},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BWz8uig4QyqPtke59pM88z","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBWz8uig4QyqPtke59pM88z.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/BWz8uig4QyqPtke59pM88z.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"ced0555a13703b843dd0521c46aae9a89353553daf48aa1e217d07fe9d141d2b","result":{"confidence_high":87.20418400824744,"confidence_low":76.43217962811622,"sample_count":null,"score":81.81818181818183,"score_display":"81.8","source_stated_rank":118},"run_fingerprint":"dd2c761c2d92563de39f63e037d0e6c93f392158ed5bceec0400eb8c4230cfa9","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6823403052eb3e76b6245e53","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_1fe1f10ccd27c2da8a8eda6e","split_or_window":"diamond-vendor-protocol-undisclosed","unit":"percent","version":"GPQA Diamond · Qwen comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"1846c54faee6b848cb72bc54ab4e26011c2a20d9aabfe9e13d9f256c149eb909","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Text Performance table; General; GPQA Diamond; Qwen3.7-Plus column"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on GPQA Diamond; prompt, shots, tools, reasoning effort and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"not reported"},"run_count":null,"scaffold":"not reported","tools_allowed":"not reported"},"protocol_fingerprint":"6d30bff99464613e41f76f72af004bc2c22f8eb7269c902c855ab4bc81a541e0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":90.3,"score_display":"90.3","source_stated_rank":null},"run_fingerprint":"00826336e4b2687d53022f24a9e5c5f20105b2d79eda4e053d46b688fb1012e6","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Text Performance table; General; GPQA Diamond; Qwen3.7-Plus column","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e292d3c46b34a34fb650826a","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:02:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:02:57.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2612d405096e203e68b56bf7efb7251ca5322888caee4f2e0781f4a18858aa63","metric_details":{"best_score_across_scorers":"0.9333333333333333","model_release_date":"2026-06-02","stderr":3.760507165451775},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Jvdw6GAXiS6T79bkpbk6Ra","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJvdw6GAXiS6T79bkpbk6Ra.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Jvdw6GAXiS6T79bkpbk6Ra.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c11db80caa95e906de1780744c86ce97eea2a244063e2f0d567b927e1b89146d","result":{"confidence_high":100.0,"confidence_low":85.96273928904785,"sample_count":45,"score":93.33333333333333,"score_display":"93.3","source_stated_rank":53},"run_fingerprint":"007184cc625c94ffaceafcf263a7fa3c763490c0ff272979ee4abadd71efa376","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9fbc57d50cc30579ef9b8901","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:02:56.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:02:56.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9f01557a75430b3c89af6a641290c98bc6442fe6a5409f7424d5c68a4f04ac33","metric_details":{"best_score_across_scorers":"0.8","model_release_date":"2026-06-02","stderr":6.030226891555273},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"o5th8Rb9WiaDbv4oEMDoFa","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fo5th8Rb9WiaDbv4oEMDoFa.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/o5th8Rb9WiaDbv4oEMDoFa.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"2373659e8ec7475db13191ce6d2e76a0549b026b89cbcfea3117945a4ce6ab69","result":{"confidence_high":91.81924470744833,"confidence_low":68.18075529255167,"sample_count":45,"score":80.0,"score_display":"80.0","source_stated_rank":110},"run_fingerprint":"5d7d5bf3bedbc08906c876e1e2625136705a849e9e9347100aa4554d1bb5208a","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6823403052eb3e76b6245e53","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-29T11:07:26.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-29T11:07:26.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6a0e0f7647c18364403e5b6238379ee04bfef31dd12399aee663c7110dd2e5ec","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.34385964912280703","leaderboard_private_problem_count":285,"model_release_date":"2026-06-02","public_example_count":10,"stderr":2.8185763989072585},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"Tt8tBXHoCEYqGLePWXkM8u","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTt8tBXHoCEYqGLePWXkM8u.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Tt8tBXHoCEYqGLePWXkM8u.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.7 Plus","thinking":null,"upstream_model_id":"qwen3.7-plus_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"089d994dba8fbbb9a94adeceadac997c7a787574abb88833801d8debec0ea2ea","result":{"confidence_high":39.910374654138934,"confidence_low":28.861555170422477,"sample_count":285,"score":34.385964912280706,"score_display":"34.4","source_stated_rank":63},"run_fingerprint":"1b45ffc1ba68972e77cff83e44e70a6b73e744de89a0358b64cb5eb8fb602647","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6823403052eb3e76b6245e53","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0d6c8634d72ad1dab91bf7f0072407f980e95533b35d203acc2ef3fc4ad269fe","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · Qwen3.7 Plus · mmmuPro"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-7-plus","upstream_model_label":"Qwen3.7 Plus"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"bc660fd761cbde72cd1c73fad6d0c94c6f629964fcb61aa07ffaae410067afd9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.462428,"score_display":"80.5","source_stated_rank":null},"run_fingerprint":"ecff31aa1fed034f998aebd13d562fe0a6b2b1c0d2476b539ea959b2eb74b953","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Plus · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb3bf508162eaffa0f3e019","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0195ba2faf3c053c54fee8a4323132e1354f95687d3b4d9b572dd4f69a5dc9d2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-02","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.7 Plus","source_effort":null,"source_model_version":"qwen3.7-plus","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"44f5ae6566dee9a53a4b7b04d9cfa376f9f9e0527eef08ee270004f94e232d01","result":{"confidence_high":18.098432,"confidence_low":4.381655,"sample_count":71,"score":9.14285714285714,"score_display":"9.1","source_stated_rank":98},"run_fingerprint":"c94b4d4976811d750e0f5982cc213e4e5b18169d34e7256163b5cb34b577d3b3","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f1212d54dd562a17b74748dc","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"80132c6926a2610957251c856555cdc20598b56caaca1e3606bcc052928c0679","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · Qwen3.7 Plus · critpt"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-7-plus","upstream_model_label":"Qwen3.7 Plus"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"741f82c0745148e582ebe5c942d266bdab50f732e351027ec7d9eb59b16c1e18","result":{"confidence_high":18.177487,"confidence_low":4.359402,"sample_count":70,"score":9.142857,"score_display":"9.1","source_stated_rank":null},"run_fingerprint":"5cdf555df80e03759ba621ed88aa3c15acfa1306d059c75d262043fd6147344b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Plus · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb3bf508162eaffa0f3e019","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:08:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:08:25.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"80f60a9d37fd647b826ef71fe5dbc1942a60ab8073884cc8db1a622ceaaf0819","metric_details":{"best_score_across_scorers":"0.24","model_release_date":"2026-06-02","stderr":4.2923469599092785},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hFRxCUnXRNMcpQzUy8GYL5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhFRxCUnXRNMcpQzUy8GYL5.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/hFRxCUnXRNMcpQzUy8GYL5.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ab260ad74790b5b2e3e8bacd00ad47f5bfb34ac72ac501858f989ff0a52a44c6","result":{"confidence_high":32.41300004142219,"confidence_low":15.586999958577815,"sample_count":100,"score":24.0,"score_display":"24.0","source_stated_rank":69},"run_fingerprint":"f7ca252e91bc06bd83c522d9d0184b5bce1c80eae605da1184be631c9df87dcd","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f114e22fba167039b349dca7","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:08:13.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:08:13.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3b816df90d3a2b1912ee5744522a592f007dfd6c6739429b99f38ed42658c5f5","metric_details":{"best_score_across_scorers":"0.09","model_release_date":"2026-06-02","stderr":2.8762349126466136},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"NGhhhxcbTVcftKdk8S7rAX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNGhhhxcbTVcftKdk8S7rAX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/NGhhhxcbTVcftKdk8S7rAX.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a013b87949b52ccb2c122e7d40df823b84e3087e38040c3fc0405a6fd90e144c","result":{"confidence_high":14.637420428787362,"confidence_low":3.3625795712126374,"sample_count":100,"score":9.0,"score_display":"9.0","source_stated_rank":141},"run_fingerprint":"77d5207201f9a41246539d897fdbc1298dcbe6d59b1e3c9907bab3a890a12a22","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6823403052eb3e76b6245e53","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_0f94a5c34a87744f357d4816","split_or_window":"qwen-vendor-comparison","unit":"percent","version":"DeepSWE 1.1 · Qwen comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"04c6230f42bf5eeb21096cc35d3653cbc80dc6f63dbe5b4d251b4d22cc409245","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Text Performance table; Coding; DeepSWE 1.1; Qwen3.7-Plus column"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on DeepSWE 1.1; temperature 1.0; top_p 0.95; 256K context; reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"148322bba9aa3bf90b760832066a5d74c8fcb793a80517a108761e22a6184493","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":14.2,"score_display":"14.2","source_stated_rank":null},"run_fingerprint":"ff3f043b1fc70ba355e54f57fc599fe01c8a4d46b8dbcd9e845c073e14db80d4","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Text Performance table; Coding; DeepSWE 1.1; Qwen3.7-Plus column","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e292d3c46b34a34fb650826a","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_232b90e5b13ff602749b9048","split_or_window":"qwen-refined-vendor-comparison","unit":"percent","version":"SWE-bench Pro · Qwen refined comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":39,"at":"2026-08-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-27","status":"fresh"},"measurement_id":"8e0f4d0bb54382f2486efaecdb42a369e38569fd05c86c464050c8d520a2a6c9","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Evaluation table row Qwen3.7-Plus; SWE-Pro column; evaluation details immediately below table"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on the corrected/refined task set; temperature 1.0; top_p 0.95; 256K context; task count, run count and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-27","source_content_hash_method":"raw_bytes","source_content_sha256":"35ca37ccc366f1ba478dab33841a2c0c18ce53fd62f291ca05341f7728b225b2","source_published_at":"2026-08-27","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"43b471b4b79bdfc0a980e402f56dea49d7cadaabd74f49a4c0c08e155c552f27","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":55.8,"score_display":"55.8","source_stated_rank":null},"run_fingerprint":"ce4d3a7952f38065aaa080a2f272c4c6b8c0f533c463f09c97b97215e1b5a99b","source":{"content_hash":"a8132c0d08dcb8936d1a9078fbf74a39072aa1d37bc0b3c112c58e2308514ae5","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next","id":"qwen3-8-flash-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen Community License model card; aggregate score facts with attribution","locator":"Evaluation table row Qwen3.7-Plus; SWE-Pro column; evaluation details immediately below table","name":"Qwen3.8-Flash-Next official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e8e250fd2a96168bc64c152b","provider_config":{"source_id":"qwen3-8-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_232b90e5b13ff602749b9048","split_or_window":"qwen-refined-vendor-comparison","unit":"percent","version":"SWE-bench Pro · Qwen refined comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"b39adc6f39237c499f6b108f11dcca0639b4c903708886372b2169dfe8507c2f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Text Performance table; Coding; SWE-bench Pro; Qwen3.7-Plus column"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen comparison run on the refined SWE-bench Pro task set; temperature 1.0; top_p 0.95; 256K context; task count and reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"4aa56bbe0e12860908d83b1e11505c57d3e76e64422ec1a69baecc6fd71cd13e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":57.6,"score_display":"57.6","source_stated_rank":null},"run_fingerprint":"e163f4af3863751d9e44e257ba126703ad231b70e75aabf64823dcaef3ec61ef","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Text Performance table; Coding; SWE-bench Pro; Qwen3.7-Plus column","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e292d3c46b34a34fb650826a","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"frontiercode","metric":"mergeability_rubric_mean_at_5","name":"FrontierCode 1.1 · Main","release_id":"release_28313b815e5f20ff3646855e","split_or_window":"main-100-hardest-tasks","unit":"percent","version":"FrontierCode 1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:18Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e72fbdfa1975b98742a97f979fe672dc53166090b1bf7b52cef73a04d0b9719d","metric_details":{"aggregation":"Mean@5","comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-02","notes":null,"upstream_source_url":"https://cognition.com/frontiercode"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen 3.7 Plus","source_effort":"none","source_model_version":"qwen3.7-plus","source_row_date":null,"upstream_leaderboard_url":null},"run_count":5,"scaffold":"mini-swe-agent","tools_allowed":"coding-agent tools; solution-bearing internet sources forbidden"},"protocol_fingerprint":"6503e291c9d7c606598f28e53137bd9e936aecfbc73279fcbd53c05a90ffbc1f","result":{"confidence_high":null,"confidence_low":null,"sample_count":100,"score":10.2,"score_display":"10.2","source_stated_rank":39},"run_fingerprint":"38103a5dcfe2c2eca409c02bc9ef44dd3f66252223cb0f0e0138daaed4ca636c","source":{"content_hash":"6990a3935a51a463b65cfd82cde38c8d4372261c74e8f3e58a903a013abc3f91","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/frontiercode","id":"epoch-frontiercode","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; FrontierCode result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · FrontierCode 1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiercode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_6823403052eb3e76b6245e53","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c0474c943882dba6f81350ed99392af2139eab30c0843719aa1ce9bd4edfc12b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-02","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.7 Plus","source_effort":null,"source_model_version":"qwen3.7-plus","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1ce3dab97d66b987eadb8844d61f963f5b913af59fd9d397acf55723fc781a87","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.4861111111111,"score_display":"45.5","source_stated_rank":122},"run_fingerprint":"765a3cdd9dbf3705b8e6b5ebb667ec43b81a72cff367b329b1fd83b4f0c77cc2","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_480536e6d05172cf6b264394","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-06T00:23:09Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-06T00:23:09Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0359ca0cd2ab1064a48308a97803a57d6f1afeb349c0bd3ebde97c79c5e21727","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · Qwen3.7 Plus · scicode"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-7-plus","upstream_model_label":"Qwen3.7 Plus"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"60ae6df7d1dcf033841b40910c20245b5983ba6f39170236d0dd1e05b711dc89","result":{"confidence_high":51.835627,"confidence_low":40.397603,"sample_count":288,"score":46.064815,"score_display":"46.1","source_stated_rank":null},"run_fingerprint":"e4ba021239ea7989def2d66937b25710f068af8ba5f236e21bf3ed261a8491b6","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Plus · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb3bf508162eaffa0f3e019","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7a528b194a95e1b973e36da26799cbc0625f5d62077bed4fc1516971c485941f","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.082348,"date_is_proxy":true,"latency_seconds":2231.75,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.614},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.7-plus"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"49e4cea51b9f436e5fc1534e06f2d9cfeea6db5b1fd0f9d14b8c5fbc108a1863","result":{"confidence_high":55.434439999999995,"confidence_low":37.34756,"sample_count":null,"score":46.391,"score_display":"46.4","source_stated_rank":59},"run_fingerprint":"0201cf384f9dc8d77127ac1b23d5b898a466be063767a6c8d5c4d1c3075f7140","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f76a19fb1cd57c1a8d315778","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7d147ab9f44c36e77c69934d62224865fc293d6ab07ee03973dd08da7a62e780","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.095849,"date_is_proxy":true,"latency_seconds":654.983,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.649},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":null,"upstream_model_id":"alibaba/qwen3.7-plus"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"ba2da6f46d8f8efdd2311a9a1b7ea8eee39b465cce90d1b6d0da259de86c83a0","result":{"confidence_high":54.081039999999994,"confidence_low":51.53696,"sample_count":null,"score":52.809,"score_display":"52.8","source_stated_rank":57},"run_fingerprint":"ae08d6009dcd664c6bc9e1f7726da689bc1e78fbb5869ddd6e05fe0ec3d8653d","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a276c3b003828a116e9aecd3","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0323c46b4a04d331dcaae318c45505807599f0607d5fa90a2578f5a4b5b4362a","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-01","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.7 Plus · terminalbenchV21"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-7-plus","upstream_model_label":"Qwen3.7 Plus"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"6c03e22fad058dd8b24e97276e3880859e9da09fe97c301d395a1884b0740d4a","result":{"confidence_high":70.521413,"confidence_low":50.661619,"sample_count":89,"score":61.048689,"score_display":"61.0","source_stated_rank":null},"run_fingerprint":"6ae866838c0330d0adce4fe08e1d23f09bd5eefc050ae06eadead8bc6b728c8d","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Plus · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb3bf508162eaffa0f3e019","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d2598ee7d5c9371e7c3dfb176a014236c11d6974944b729895c95c5cb7185f06","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-01","source_locator":"Leaderboard models table · Qwen3.7 Plus · tauBanking"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-7-plus","upstream_model_label":"Qwen3.7 Plus"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"f25bc24e1a4777611dd1e305f8739a26c00d5626a3c43ba409c46c1216314c27","result":{"confidence_high":26.285821,"confidence_low":11.239961,"sample_count":97,"score":17.525773,"score_display":"17.5","source_stated_rank":null},"run_fingerprint":"6327efeb854e122bd4004bd35153754f1fc4d214ad34d6197d3ebbd19cbd6f5d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Plus · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb3bf508162eaffa0f3e019","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bb62286d77628cfe4d969cd92b972ec531efbaee775d8358104bf8e238e979a1","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":21722},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Qwen3.7 Plus","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"367052c886b05e741791a4b6bcaab039c58fec1ce36ac01093e4dd3528256f69","result":{"confidence_high":-0.056984555835833235,"confidence_low":-0.08610369957455825,"sample_count":1144063,"score":-0.07154412770519575,"score_display":"-0.0715","source_stated_rank":43},"run_fingerprint":"846d80c8753e1a588e7e0f5b737517eb41a96388a6d8384faa806c18b691cf4a","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_73a5d0395f902cb3229076c8","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"61c81dec8385c9276f4ae1070362cf41150789c9c701373d60b688510364d9dd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-06-01","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Qwen3.7 Plus · gdpval"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-7-plus","upstream_model_label":"Qwen3.7 Plus"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"4245ba977968f8c9185a40930363424b93f36274177fa77c4a29d0df30c6dde6","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":756.04,"score_display":"756","source_stated_rank":null},"run_fingerprint":"5c468970cbaf18eca856d48d429c776f039b7b69711e906916437ef2611557af","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.7 Plus · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb3bf508162eaffa0f3e019","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"computer-use-agent","higher_is_better":true,"id":"osworld-2","metric":"binary_accuracy","name":"OSWorld 2.0","release_id":"release_15c5525fe39f33bc181dfb69","split_or_window":"binary-accuracy-500-step-budget","unit":"percent","version":"OSWorld 2.0 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:06Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dc4fb76dc97764a2835df62c0d1c7bc4feab1f66bc9f4db396692c01d61e55f9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-06-02","notes":null,"upstream_source_url":"https://osworld-v2.xlang.ai/"},"model":{"id":"alibaba/qwen3-7-plus","name":"Qwen 3.7 Plus","provider":"Alibaba","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen 3.7-Plus (thinking)","source_effort":"thinking","source_model_version":"qwen3.7-plus","source_row_date":null,"source_scaffold":"standard","upstream_leaderboard_url":null},"run_count":null,"scaffold":"standard","tools_allowed":"computer-use actions in the OSWorld 2.0 environment"},"protocol_fingerprint":"6148a7636d450b51ea486e87b225d1c3a61dcb0e7064ecec4c1bbcf6458770ee","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2.8,"score_display":"2.8","source_stated_rank":16},"run_fingerprint":"268f32f9be4758b609a4fcfc71c81f5a8e4a8d94be06dc81b62e13efb1d2e26d","source":{"content_hash":"8b464e7081e7ad4a35167078f40b9f7e892a9bc3825fbcbc09d6eb28a3208811","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/osworld-2","id":"epoch-osworld-2","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · OSWorld 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/osworld-2"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d4e18270aaf1ff5ea39c0e8b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d1988b8739080f176eeeb88512666e130cef58c7da13a4518bfc415d39d69954","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-18","source_locator":"Leaderboard models table · Step 5 Preview · intelligenceIndex"},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"step-5","upstream_model_label":"Step 5 Preview"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"90efb4ef8b320a0c4d958b83ee98de3f764fe2addf7926486d713f506160d578","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.734305,"score_display":"43.7","source_stated_rank":null},"run_fingerprint":"8c4737f5716f2e0ba1270a1a0ce4858a8a99dfb6f4de835acca1d3b978306d13","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Step 5 Preview · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6163e1e4e2e25c4ca85b2f88","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"595ddf557b82710b022e5a9e230334fff7e9d053c832c74f2e817c530c472884","metric_details":{"license":"Proprietary","variance":31.924111740056144},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1464.7537138352343,"confidence_low":1442.6055617103582,"sample_count":2701,"score":1453.6796377727962,"score_display":"1454","source_stated_rank":74},"run_fingerprint":"26c7a730f1d1ee4cc8c536cc50932a61b53c8200552e124b3e024ebbff4a7f11","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9f6dcc62a58559d366b17164","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6a70492cbf5952be82a22d1ea521b165ab7b54ce44a6a60603e07dd566cc4069","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-18","source_locator":"Leaderboard models table · Step 5 Preview · hle"},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"step-5","upstream_model_label":"Step 5 Preview"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5f0b950d61fa78586cb81ccacfc714c4697e68793d44c71dfb58ecd3e10a8164","result":{"confidence_high":48.58698,"confidence_low":44.381979,"sample_count":2158,"score":46.478221,"score_display":"46.5","source_stated_rank":null},"run_fingerprint":"7b738cf74429182dc534d5978f7a50719fa545a9d1566184c0a3d6e150546505","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Step 5 Preview · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6163e1e4e2e25c4ca85b2f88","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cc198ca41fcf775b16efdf3f55dc64d6da95922919c3ea270af29081052aa9c6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-18","source_locator":"Leaderboard models table · Step 5 Preview · mmmuPro"},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"step-5","upstream_model_label":"Step 5 Preview"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"89ad6c4a1d548cb05bb7933349c482ee46b07064b2837e1b621adb142afff549","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.358382,"score_display":"76.4","source_stated_rank":null},"run_fingerprint":"423ff8d15d7a04e52e50b33bc542b755c568b5c72022ebff94316d2b42b59425","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Step 5 Preview · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6163e1e4e2e25c4ca85b2f88","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"dc610008de7a05fb863fff0dacc6c36a463a0d6d6fb0dba2cf420f9b7d49ea26","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-18","notes":null,"upstream_source_url":null},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Step 5 Preview","source_effort":null,"source_model_version":"step-5-preview_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"ec00206f4f5aaf243389d3edc7f85f99eb2c36665f4e830f224028afa3afd69e","result":{"confidence_high":31.678669,"confidence_low":13.027407,"sample_count":71,"score":20.8571428571429,"score_display":"20.9","source_stated_rank":48},"run_fingerprint":"b67361f12da38eb3833a0f1947830e969ec9c7fd62c23f764a8ec5fc695b7acc","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8ae9b91e96bb1630eea01fe7","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7757e3fd5abad071f08acbefcbdccb224cec6a06113b6fcbb3279b3f5b29fbbc","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-18","source_locator":"Leaderboard models table · Step 5 Preview · critpt"},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"step-5","upstream_model_label":"Step 5 Preview"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"95d0dc08837b45a5b96684d142fa9e75d73ba49bc57365ac5ae601f3374db460","result":{"confidence_high":31.763497,"confidence_low":12.983096,"sample_count":70,"score":20.857143,"score_display":"20.9","source_stated_rank":null},"run_fingerprint":"06da3e1f82b11d2908e815e2483ba0430dc15a12873b33dcf440d8ee68121c8f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Step 5 Preview · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6163e1e4e2e25c4ca85b2f88","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dd6d94b809c93b7add4e94bd2c27111927ac4a6c54bfcd5a58061aea4976afcd","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":2.025817,"date_is_proxy":true,"latency_seconds":4820.902,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":6.69},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":1024000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":null,"source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":0.95,"upstream_model_id":"stepfun/step-5-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"7867eff302f5343cb3895d031748037cb8333d4837e8c59b9aa88c5b6a73d368","result":{"confidence_high":51.1684,"confidence_low":24.943599999999996,"sample_count":null,"score":38.056,"score_display":"38.1","source_stated_rank":34},"run_fingerprint":"dd489ef7fbca002914d2637fb0e5c1adc6b24ae8b76abf71b0995820ddcd1861","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_80831aa0cb0d312c1beaf624","provider_config":{"source_id":"vals-ioi","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ad3327b3afdea0075dd2db0d194d873dd0ae0031c1610e4f060772b93edc6230","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-09-18","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Step 5 Preview","source_effort":null,"source_model_version":"step-5-preview_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"962f818b99d0ed7640b7d946bb09198eb356bcce789b2edf6bdc6f821c7a3019","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.912037037037,"score_display":"58.9","source_stated_rank":13},"run_fingerprint":"362d5b71c57f45eb554deaa395d667f5a2143a2f4ec5ab04e054264ec89eb41e","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9844486f407e9d5626c38e1b","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c94478fa5ffd1ceeba8661e468921d108d6f0568519e894788c1741aaff0f94c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-18","source_locator":"Leaderboard models table · Step 5 Preview · scicode"},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"step-5","upstream_model_label":"Step 5 Preview"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"f7d26bc478a81e02adc7c506e9043543efbc66f463becb732994d4772ba619a4","result":{"confidence_high":64.440656,"confidence_low":53.148794,"sample_count":288,"score":58.912037,"score_display":"58.9","source_stated_rank":null},"run_fingerprint":"52a8f76fc27c43820b6a51d2e561ac48b136f9f3c77e5bd4074468faab479c25","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Step 5 Preview · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6163e1e4e2e25c4ca85b2f88","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"de5d9c743f4d6be4a41e0acce5872f7f427934fe0cf68b988153dfb953d98e48","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":5.856257,"date_is_proxy":true,"latency_seconds":5466.358,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.267},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":1024000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"stepfun/step-5-preview"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"1577249af88f9fe6fcbecf9c6374a1fa67b8fe42e8e6a8841929ce868ff7e421","result":{"confidence_high":78.76232,"confidence_low":62.03568,"sample_count":null,"score":70.399,"score_display":"70.4","source_stated_rank":34},"run_fingerprint":"76b0fa9ea6274b080ba5b49baa590d51542d79b99516715d19bbbfc600550822","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_404128e4ce18921deeb03b6a","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c99a8c34f78d54e24977b931c0e566402542edb19f4df7f431d5c070c16d1c1b","metric_details":{"license":"Proprietary","variance":42.527153538297625},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"step-5-preview-high","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d098b2cbd8b689702aed1567521c6515307342e649bf8fd7f8b4c98f19e533d9","result":{"confidence_high":1582.9324901374748,"confidence_low":1557.369524027987,"sample_count":2506,"score":1570.151007082731,"score_display":"1570","source_stated_rank":33},"run_fingerprint":"142bacc612ca6a09cf5ce2c9c58b5950e76a266b7eccf7e5791e5295b98556ea","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_eb70fd26c572537944129f9f","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"254f9a565abc90ac97128bb1778751f3aa39ef6bf4c8eb13bec53759b46fea4e","metric_details":{"confidence_level":0.95,"license":"Proprietary","session_count":9453},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Step 5 Preview","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"2cc174740bd01b4604ee3f83e3e96bf5325a724517688a1d42e36135332324fd","result":{"confidence_high":0.019527238277087543,"confidence_low":-0.009863944134801792,"sample_count":1125598,"score":0.004831647071142874,"score_display":"0.0048","source_stated_rank":30},"run_fingerprint":"bddc09224c86c2d5c562284defeda9184479115f47b174bf88242801092765a5","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_73b151451dd1e7342d7ff672","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8ebe6dcd02afa4ccf12b8ba670418d71235ce648dfbe70d0409a9b3c8abc0ee9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-09-18","reported_confidence_interval":"-31 / +31","source_locator":"Leaderboard models table · Step 5 Preview · gdpval"},"model":{"id":"stepfun/step-5-preview","name":"step 5 Preview","provider":"StepFun","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"step-5","upstream_model_label":"Step 5 Preview"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"745ca87b8cb4614df551cab6cc93f83d5631f853513ce73f0e56900a0af99f9e","result":{"confidence_high":1597.48,"confidence_low":1534.76,"sample_count":220,"score":1566.12,"score_display":"1566","source_stated_rank":null},"run_fingerprint":"924d4ea64cce74dea5e552609cdb8a298f4af21cf1fdfa1d7f68210f227b5788","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Step 5 Preview · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6163e1e4e2e25c4ca85b2f88","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9618e7611fda87231cb2417935fe5eebeaaa50d7e43e611d13a88a794e23ed41","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-03-17"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-5.4 Nano; model release: 2026-03-17","source_accessibility":"API access","source_model_name":"GPT-5.4 Nano","source_model_release_date":"2026-03-17","source_model_versions":["gpt-5.4-nano-2026-03-17_high","gpt-5.4-nano-2026-03-17_none","gpt-5.4-nano-2026-03-17_xhigh"],"source_reasoning_efforts":["off","high","xhigh"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6b46642c206b5cd25b78f89374c939ea2cc6f95f2d769a0eecb659e1a43bcba8","result":{"confidence_high":147.7,"confidence_low":143.21,"sample_count":null,"score":145.8,"score_display":"145.80","source_stated_rank":78},"run_fingerprint":"10975b4ea622db862105f0b53c5e4c298447ab31b73da1bc8b7020aaeed418e9","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c58cd8aea3620eea4e04e0b2","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"19e6cdac5d09f8026fc0c27669327e0619868cbe6532041c4afa6fcebf85c8d2","metric_details":{"license":"Proprietary","variance":3.735497845891428},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1405.1118877910892,"confidence_low":1397.5356720735206,"sample_count":62249,"score":1401.323779932305,"score_display":"1401","source_stated_rank":158},"run_fingerprint":"230215f7d4f6c03abd73aca582bde07163dc7835af87171756f49c23a977821b","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"8399cfa1d7e8df2f8088dbe672dd759a735b6863d3e6c854f4a665895b56e4c5","metric_details":{"category_scores":{"Agentic Coding":46.76766666666666,"Coding":70.83850000000001,"Data Analysis":67.64266666666667,"IF":67.20475,"Language":62.507000000000005,"Mathematics":90.977,"Reasoning":81.09700000000001}},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.57636904761905,"score_display":"69.58","source_stated_rank":55},"run_fingerprint":"5a462a8e90f963c202f75a44840d64a106df75bce86e3e4242b865c35c6a305f","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_0bdad69e8278c95ccb05d676","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:28:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:28:50.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef018577813091a81a4a06a9a71f577600dfce78e52c12f0c5f02a50d63568d7","metric_details":{"best_score_across_scorers":"0.117","model_release_date":"2026-03-17","stderr":1.0169287802713345},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"3QBsxKkbF2R2LEXTd6HDrJ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3QBsxKkbF2R2LEXTd6HDrJ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/3QBsxKkbF2R2LEXTd6HDrJ.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"40c85021eb5d68a8360d0cf162dc8111f4c27837683acd2c7671760bb97b1290","result":{"confidence_high":13.693180409331816,"confidence_low":9.706819590668186,"sample_count":1000,"score":11.700000000000001,"score_display":"11.7","source_stated_rank":82},"run_fingerprint":"7a8077ca7b457c26165d698b82e357a9b26297c79189f6407eacf1016a488475","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3f7a6d7e7d138764420f5a73ff340aa94686c006431632f6294ef5385807984f","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.001565,"date_is_proxy":true,"latency_seconds":29.498,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.417},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":76.768,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-nano-2026-03-17","zero_shot_accuracy":78.283},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"9769bc617cc0d2b741fdbe43ae786bd208dbb0507954029160977f93d5d568d2","result":{"confidence_high":82.26332,"confidence_low":72.78868,"sample_count":396,"score":77.526,"score_display":"77.53","source_stated_rank":78},"run_fingerprint":"e5489752c15316ed2cfd23884e46de068baacfd2719cd9cdade488e9196b2586","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":173,"at":"2026-04-14T15:07:20.513Z","basis":"evaluation_started_at","evaluated_at":"2026-04-14T15:07:20.513Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b460afa9d7a9a94c459d3863b4c541e4dfb6358990543b0ae84316368a3cb4a1","metric_details":{"best_score_across_scorers":"0.7847","model_release_date":"2026-03-17","stderr":2.44},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"fZB3qGSohAVYzN97mniuUp","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfZB3qGSohAVYzN97mniuUp.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/fZB3qGSohAVYzN97mniuUp.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"9acb231c809f3e6c6f4687e44ef910c409af60b69d92cf9ef3fed912899cae1d","result":{"confidence_high":83.2524,"confidence_low":73.6876,"sample_count":null,"score":78.47,"score_display":"78.5","source_stated_rank":137},"run_fingerprint":"26f0bfa0f42b5935fb3516b93b199a25d34ee6e102d99456701ad1354a11b985","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:00:23.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:00:23.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"df2e85cf372b637e1de28dbdcbbadae3ad4360812906a474368feb133c1a3993","metric_details":{"best_score_across_scorers":"0.7222222222222222","model_release_date":"2026-03-17","stderr":3.1911782267135482},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Nq7w5iKALDSuT5oQdn5m8f","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNq7w5iKALDSuT5oQdn5m8f.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Nq7w5iKALDSuT5oQdn5m8f.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"b1407fbe20d6541ac91adcbd08f6e2faf0b64812b48062967e73ad139e10af5d","result":{"confidence_high":78.47693154658077,"confidence_low":65.96751289786366,"sample_count":null,"score":72.22222222222221,"score_display":"72.2","source_stated_rank":172},"run_fingerprint":"9d8a5b8865a3190a66c701c46733a32e2b1d46a4d413a3e244d4c3cd008cce9b","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c8fdf9bb82a19ed421dab3da","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:01:43.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:01:43.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"965575db8f0838a1dbbc5d023a74a26427da1eaafb11456d1e5dd6a47dc8208f","metric_details":{"best_score_across_scorers":"0.5555555555555556","model_release_date":"2026-03-17","stderr":3.5402943770953677},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gLewKMc7ttCD2moUrgAyYX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgLewKMc7ttCD2moUrgAyYX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/gLewKMc7ttCD2moUrgAyYX.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"36d65db9133d8bfad84115592a5d9162e4e02ce6793df8f4a63b8f9ff40ccf64","result":{"confidence_high":62.494532534662476,"confidence_low":48.61657857644864,"sample_count":null,"score":55.55555555555556,"score_display":"55.6","source_stated_rank":223},"run_fingerprint":"765b11667a64dbafbc77712c758098582e15150244d837cdfab3075677443200","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c34d174d18e7a8b7849cbd79","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"67595f21bb7cce8cb025eb90cdc096777d559f149052d1cf1d38289c90260740","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.000396,"date_is_proxy":true,"latency_seconds":8.21,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.433},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-nano-2026-03-17"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"588068a64b005904738b052532fdf7cacdab2ac09e90a171403d440db4da2739","result":{"confidence_high":78.02068,"confidence_low":76.32332,"sample_count":null,"score":77.172,"score_display":"77.2","source_stated_rank":108},"run_fingerprint":"f55fc2bd258761ebb651882b6492cb0e03af2c3bc924437d9043f7c674a01fe9","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":173,"at":"2026-04-14T15:08:33.525Z","basis":"evaluation_started_at","evaluated_at":"2026-04-14T15:08:33.525Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"55a10e68ce247f99fd8fb7f00154b69f9167c0a35a7a0c40f67aef65f3da0d4e","metric_details":{"best_score_across_scorers":"0.8778","model_release_date":"2026-03-17","stderr":3.94},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jVv9m8DjkdpZaLPdUzAnG9","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjVv9m8DjkdpZaLPdUzAnG9.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/jVv9m8DjkdpZaLPdUzAnG9.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"533f1ef8c0aa7368cc8ce278ba524911f1b9cdf50e3b24c5b3d290e6cbb3e9ee","result":{"confidence_high":95.5024,"confidence_low":80.05760000000001,"sample_count":45,"score":87.78,"score_display":"87.8","source_stated_rank":79},"run_fingerprint":"46bc88b9134082eb69dcd7fd5af31c023a33477f2dd5e0590fe64b6071cd37ac","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:00:00.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:00:00.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bf23e439cf7c8d18de6e2ca25b08834169ee4fa7a4d9d05a058e0302b52da78e","metric_details":{"best_score_across_scorers":"0.6888888888888889","model_release_date":"2026-03-17","stderr":6.979205927323111},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bxvsuZ4Xpi3gZ4rsA8T7kV","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbxvsuZ4Xpi3gZ4rsA8T7kV.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bxvsuZ4Xpi3gZ4rsA8T7kV.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0f4f3ca6fd83d7378a802cdbe5a8b019cdde9ab88782d5bbba9b8dcb710240d5","result":{"confidence_high":82.56813250644218,"confidence_low":55.20964527133559,"sample_count":45,"score":68.88888888888889,"score_display":"68.9","source_stated_rank":140},"run_fingerprint":"172694aca3cca6638fc9d8e7ec108c4c30de5136ef67be61e6089755454b6d6a","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c8fdf9bb82a19ed421dab3da","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:01:03.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:01:03.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fb77b1b59f0e48c5ad8c38069d878bce0cc75c0c5449e3e69aa99a5ca4811d34","metric_details":{"best_score_across_scorers":"0.4666666666666667","model_release_date":"2026-03-17","stderr":7.521014330903548},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"eUR9o3bvU2Yu3KY5w5uk4j","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeUR9o3bvU2Yu3KY5w5uk4j.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/eUR9o3bvU2Yu3KY5w5uk4j.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3e7ec78e3251e640ee52a71baa8ccd9d4f8b26e6db56a5c07885a8cf17f4082a","result":{"confidence_high":61.40785475523762,"confidence_low":31.92547857809571,"sample_count":45,"score":46.666666666666664,"score_display":"46.7","source_stated_rank":193},"run_fingerprint":"e98f803f67fc2ce4a1748992a9238d6296b675a041978c38139571b048ece492","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c34d174d18e7a8b7849cbd79","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-12T16:48:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-12T16:48:06.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"693ff3ca83066ce5a9322e75324760f9615da9e9af60da12e8d2862bd680062f","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.44912280701754387","leaderboard_private_problem_count":285,"model_release_date":"2026-03-17","public_example_count":10,"stderr":2.9515543245521676},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"iKtopjTjyCWZqJaEjcc7ZL","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FiKtopjTjyCWZqJaEjcc7ZL.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/iKtopjTjyCWZqJaEjcc7ZL.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 nano (high)","thinking":null,"upstream_model_id":"gpt-5.4-nano-2026-03-17_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"93d550bd4f0659ec8d3b3eb5c3b3db2737270435e50ab1c146000095e38607fd","result":{"confidence_high":50.69732717787663,"confidence_low":39.12723422563214,"sample_count":285,"score":44.91228070175438,"score_display":"44.9","source_stated_rank":52},"run_fingerprint":"5297f35c349d3272dd5841809bebc242be50ac57731284af55d4eff890d32fc5","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-28T12:28:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T12:28:58.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6434a326521436b41948f2983678c86914a94b597e77a7d787f1c89c3621f7ce","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.20350877192982456","leaderboard_private_problem_count":285,"model_release_date":"2026-03-17","public_example_count":10,"stderr":2.389037106974561},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["low"],"epoch_id":"BNSjwbcpNSaDXHF7gC9WoN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBNSjwbcpNSaDXHF7gC9WoN.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BNSjwbcpNSaDXHF7gC9WoN.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 nano (low)","thinking":null,"upstream_model_id":"gpt-5.4-nano-2026-03-17_low","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"d402c81761dfeef32e9a1cf94c5755b8f8899aed5b24c33feb1f0146349bf855","result":{"confidence_high":25.033389922652596,"confidence_low":15.668364463312315,"sample_count":285,"score":20.350877192982455,"score_display":"20.4","source_stated_rank":83},"run_fingerprint":"06111573c131c160723e3bac683713b44d8ee3953ec575b503e0fefbd87bc0b6","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c8fdf9bb82a19ed421dab3da","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-28T12:29:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T12:29:12.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e4d6f7b48a9dd6cb620b0a4568b74be234e293e307e440b4efc231a585697c34","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.0456140350877193","leaderboard_private_problem_count":285,"model_release_date":"2026-03-17","public_example_count":10,"stderr":1.2380896644748405},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["no thinking"],"epoch_id":"KDQZWiPXB2VXc4bj4GJQW4","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKDQZWiPXB2VXc4bj4GJQW4.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/KDQZWiPXB2VXc4bj4GJQW4.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 nano (no thinking)","thinking":null,"upstream_model_id":"gpt-5.4-nano-2026-03-17_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"8ee987ae44d28d40fb6798e359366f2e736a5545f1806581a17f0954a2086928","result":{"confidence_high":6.988059251142618,"confidence_low":2.1347477664012424,"sample_count":285,"score":4.56140350877193,"score_display":"4.6","source_stated_rank":108},"run_fingerprint":"39ae582efd5da252e5a70c642e8a863450a804d0e01498033d436f8a8809f45e","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c34d174d18e7a8b7849cbd79","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dac412180b5ddad4ceee32516261e08ee3af6cac31771cccf56ace2d1b8b97c3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.16,"model_release_date":"2026-03-17T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4-nano-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-4-nano-xhigh"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"01ad944d01ddecc00a82f5739b0aa8928b7e1d102db1bd4cb03edeee749afc04","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5.6899999999999995,"score_display":"6","source_stated_rank":160},"run_fingerprint":"45e85568cd5bdb2579756a09644cb447ee5252107b1f7be32019cc1e0740878b","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_0bdad69e8278c95ccb05d676","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5bdbb54017ac0672a508a1eed772582b1636388e9473663f74ba57a8f83a6dc6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.13,"model_release_date":"2026-03-17T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4-nano-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-4-nano-high"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cb09557bf38a36d03f3a3780f67e9aa1551a92e276eb0f0394b0077d3d4fb05a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3.61,"score_display":"4","source_stated_rank":180},"run_fingerprint":"deeb3fe240f79da55d27dc73e58147a8ed2ff666905f3c2c1a93d427b1aa0833","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7c4b16879a1939e475a221cf2b3a2df30670f0c62a68d68b015ff86d6b9be4dd","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.06,"model_release_date":"2026-03-17T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4-nano-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-4-nano-medium"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"dc5cbf1cdc0e27e1bb40cefceaf9896f69ff127be691822246f6e948c3d45ef1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.94,"score_display":"2","source_stated_rank":199},"run_fingerprint":"cec722fca7e5eb164efed8c06ce19b9f90b4014cce57ffe4bf147963e0aabe7e","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_bf35ad9aa2008c0d2229cd8c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"578459ab2e4ebb738dac7005f7b46fc878352612c25b1b40f3b57dba70fbb3b9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.01,"model_release_date":"2026-03-17T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gpt-5-4-nano-reasoning","model_type":"CoT","upstream_model_id":"gpt-5-4-nano-low"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fd9a65c89e87dc3195e6bbbbe161567ea89e28986a8c6c0ffcc921d9dfe4f93d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.53,"score_display":"2","source_stated_rank":209},"run_fingerprint":"b6717a8b9fe5444b37b464282bd0b48ed7aa5542f67de076521347d3d8f4f41f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c8fdf9bb82a19ed421dab3da","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"145fdcbb1bea307866036fca3c581b2ca5cfbb2201fbd49e0d4797c13480e130","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.16","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (xHigh)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"042f2845c0eaa4c5aaa0cdfaea153efd4c2d1335dd1df56982804a7e6c5c8736","result":{"confidence_high":8.583703,"confidence_low":3.731986,"sample_count":360,"score":5.6899999999999995,"score_display":"5.7","source_stated_rank":155},"run_fingerprint":"841f727d85e0033c19a7d20931778f754df96526c26bb672a5f293b2d298168d","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_0bdad69e8278c95ccb05d676","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8fcef89eedf6c0bfcde36b6b7e4fba62d10d763ab7d458ab289bd1ea5ea12f61","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (XHigh)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"adb8ba01e7b637f741fbdada84194ac6b6e105a2ef52d6153c868e8ee153bf36","result":{"confidence_high":8.583703,"confidence_low":3.731986,"sample_count":360,"score":5.6899999999999995,"score_display":"5.7","source_stated_rank":155},"run_fingerprint":"20505bc30af61ff33fedfe6746bd692e22adb7f4cd3e2143b588f22b465cf0bd","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_0bdad69e8278c95ccb05d676","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6afacaa9e1ea3f1238503fbb2856e1a108a98dc9a950e50b8adbcfe6e2d32a7a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.13","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":7,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (High)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"51e33f473d2af91782839a5bb29c8eec3808d2a1e0296c568aa14c996bbe29e5","result":{"confidence_high":6.078166,"confidence_low":2.121446,"sample_count":360,"score":3.61,"score_display":"3.6","source_stated_rank":175},"run_fingerprint":"858fb0de29329e57b7614b9ea7847898b8d8086e1402be4aef71c74f8c917fe6","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"83f23314a5e7c11a56c503448722de8c98f1d5e1e2a76370cf415aff2f7f3a3d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.06","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":8,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (Medium)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"394449beda4614eef970937f6a3c2ad7cfaec9d4e191200eda91a8f07d6ba944","result":{"confidence_high":3.952794,"confidence_low":0.942084,"sample_count":360,"score":1.94,"score_display":"1.9","source_stated_rank":193},"run_fingerprint":"af3f8cb9f19398ed98873930e23604fb42e5a466c2c8d32f43e48b41c2209fd8","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_bf35ad9aa2008c0d2229cd8c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6bf9a53a362d72819a2206978d50d42e8b6f7e1efe38beae8f40e987d5f372c2","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.01","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":9,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (Low)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a38e6e8d76608d95450402b9e4b0976ba7389c24af971f1cd71a0012f55a5cc7","result":{"confidence_high":3.402882,"confidence_low":0.680654,"sample_count":360,"score":1.53,"score_display":"1.5","source_stated_rank":202},"run_fingerprint":"86c933a2e3a3069a4a7698a90baaaf52f55739c2db3fd73f02c3cf92669e0561","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c8fdf9bb82a19ed421dab3da","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":114,"at":"2026-06-12T16:48:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-12T16:48:06.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4157de819111bdc7c7780a9f8b86af388594df0f8251cf8613fe1626d09cf2e4","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.12195121951219512","leaderboard_private_problem_count":41,"model_release_date":"2026-03-17","public_example_count":2,"stderr":5.173952057462543},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["high"],"epoch_id":"SF6FYnnKztPCDWQc4v4EJa","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSF6FYnnKztPCDWQc4v4EJa.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/SF6FYnnKztPCDWQc4v4EJa.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"GPT-5.4 nano (high)","thinking":null,"upstream_model_id":"gpt-5.4-nano-2026-03-17_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"d893fe105e20ffada7ffbe0fcf3bab7fdadcc1d84a7dc9423bf50cae4db895aa","result":{"confidence_high":22.336067983846096,"confidence_low":2.0541759185929287,"sample_count":41,"score":12.195121951219512,"score_display":"12.2","source_stated_rank":54},"run_fingerprint":"a534fda00b79bf949088415ad21f331404e4e772daa2b1b8375307c204a9f777","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f0a8537bf722188d10a12ab61828e388f487dc5fc896e741ab791893be6b81da","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.000844,"date_is_proxy":true,"latency_seconds":21.907,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.06},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-nano-2026-03-17"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ad82d292f74979ef18db7c9d59c9b292fa04629f43b25c2cf073d244e962feb6","result":{"confidence_high":75.6616,"confidence_low":71.5064,"sample_count":null,"score":73.584,"score_display":"73.6","source_stated_rank":59},"run_fingerprint":"f06761e6bd4179687d2235257a7f92efb5de768a86ac84b05ae5ad2ce9612559","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0a82f41f4f69d1faf78de392b627a637a5bfc4971ec011d87aa5d5db7627f17b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 nano (xhigh)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"25bafaeff44f9031e2215ad29d9832ff31cd14c4fd8728e9c5e001fa17a95b88","result":{"confidence_high":18.234831,"confidence_low":4.454351,"sample_count":71,"score":9.253061224489791,"score_display":"9.3","source_stated_rank":97},"run_fingerprint":"1f7efc34a25412f5fe8f2c785198591ccbbac6c773dea1e81ce097b3758e8806","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_0bdad69e8278c95ccb05d676","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":173,"at":"2026-04-14T15:10:08.857Z","basis":"evaluation_started_at","evaluated_at":"2026-04-14T15:10:08.857Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"00e6855e11985d8895279f117685c2f554307210271cc1333501d39cc9aa1c89","metric_details":{"best_score_across_scorers":"0.3","model_release_date":"2026-03-17","stderr":4.61},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"49RKnpqVsfDx6CvHF42Qqq","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F49RKnpqVsfDx6CvHF42Qqq.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/49RKnpqVsfDx6CvHF42Qqq.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5b76c182fde5c40e14e622298b2bda44ae609c2a3bddae20734b7b2ca2736739","result":{"confidence_high":39.0356,"confidence_low":20.964399999999998,"sample_count":100,"score":30.0,"score_display":"30.0","source_stated_rank":51},"run_fingerprint":"695bbe5e60c681ca22a758dd22f4100601ad7a03e1a1cb48092dc7f8f8c5ecc4","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:00:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:00:30.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7f6d7d1c7560b6c5cf881318343fd5f7128169981978cc720c1cbc6a59ebec61","metric_details":{"best_score_across_scorers":"0.17","model_release_date":"2026-03-17","stderr":3.7752516806863685},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gBwdGAwNy4HR6gVM88srfd","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgBwdGAwNy4HR6gVM88srfd.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/gBwdGAwNy4HR6gVM88srfd.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"926bd6d28e64425e5060ee9bcfc988215628bb6bb41d2e183990d1958b2b75b0","result":{"confidence_high":24.39949329414528,"confidence_low":9.600506705854718,"sample_count":100,"score":17.0,"score_display":"17.0","source_stated_rank":104},"run_fingerprint":"e42061d00624cbabe3c3ea0bed90a3a7504f36d830704be3f3075544651f6423","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c8fdf9bb82a19ed421dab3da","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:01:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:01:58.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1f532927b3ca753be25cc31a2481d36755d6a436629c54c8276cc5036b2135b2","metric_details":{"best_score_across_scorers":"0.03","model_release_date":"2026-03-17","stderr":1.7144660799776528},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gNmoRrvxYLs9WNSaSSDgH7","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgNmoRrvxYLs9WNSaSSDgH7.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/gNmoRrvxYLs9WNSaSSDgH7.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5db9487ccad13d41e2116acdc6bcbd045dc6c61ff24047bbfc2b49821d6dc53e","result":{"confidence_high":6.3603535167561995,"confidence_low":0.0,"sample_count":100,"score":3.0,"score_display":"3.0","source_stated_rank":180},"run_fingerprint":"6b65ba3703f98397ab6499d28bdbcdfcb5a1673ca1c390b38512f924fa4d14e3","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c34d174d18e7a8b7849cbd79","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"25036f9fac2551124af0303adaa017a8c54201257fd2b6030cfc70f4f1a26f7e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.13","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (xHigh)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"6639afb33b297a63129ad1db4133d6f05051363ec0cc4f04a095f163e8058946","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.5,"score_display":"51.5","source_stated_rank":166},"run_fingerprint":"860af363418c83582d532c0065d691ff88af5697d084c8cbd733ad935c72e007","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_0bdad69e8278c95ccb05d676","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4a57493ec7c5840715ead07510352683164394d963c99d2533b2fb8947de955b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (XHigh)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"3925a6502125a0997cb7128ed76379c95cb9796cbd2549e175c28d0e0890cb9d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.5,"score_display":"51.5","source_stated_rank":166},"run_fingerprint":"68d57cc6f19a102d565f5a6b320e6284ec675db09b69c7bd527f492d99cbe65f","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_0bdad69e8278c95ccb05d676","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5cabacd2fddf3b451926e122e181c2b040dda8c9f0fca2ac976e07ad5682a8aa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.07","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (High)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"db324980b9c9300c6c8d69ad0948add76c59c210c478e32f55289c0edbceb245","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":38.17,"score_display":"38.2","source_stated_rank":184},"run_fingerprint":"26fd1f911fa5342b67b13c5dd996d25ef484b66b48cf369aa4e285ebb7f94832","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"27991602b406eab5c0293313d15d1b23df9c04953dffee9d7bbacc4d569294cf","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.04","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (Medium)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_medium","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"fa1976cfefb16095129f2587a2f85b9604c252e0d44b563a9653b2a1bf9fe916","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":33.0,"score_display":"33.0","source_stated_rank":197},"run_fingerprint":"3ea1f66bceb9dfe13a0b0a5ec74476a13db75ac0a61970c95b8a3e40122647a9","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":3,"is_provider_default":false,"normalized_effort":"medium","profile_id":"profile_bf35ad9aa2008c0d2229cd8c","provider_config":{"source_label":"medium"},"provider_mode_key":"Medium","raw_label":"medium","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8084d1877c041d159c4232406c97186cde369f29e58ccddfbd0b08f412a2ba8d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.01","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://arcprize.org/leaderboard"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 Nano (Low)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_low","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"fcdaca2735df502599583859371f6b8584b7d1bb49343857729c7b64b41e59ea","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.33,"score_display":"18.3","source_stated_rank":228},"run_fingerprint":"ead796442edc2cfd2fee1731b738bf9391d0e22f1f95128076dfd904a7e52fbd","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_c8fdf9bb82a19ed421dab3da","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c90ca6ec352ddc8045832d669525a650b992fff6be4988d883bc237108079543","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.097712,"date_is_proxy":true,"latency_seconds":262.552,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.055},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-nano-2026-03-17"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"3d78091c3aab9cf781d8fd993904aa46333350f5cb371f8e28ae9cd8be44a058","result":{"confidence_high":73.8278,"confidence_low":65.7722,"sample_count":500,"score":69.8,"score_display":"69.8","source_stated_rank":64},"run_fingerprint":"660ee071ace73ccd6f2d852025639fd66668cf7560eee4f0da30ae7cf352cc43","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9810c68fc8700c28262cb521f9d3fbd41a4afc23321418245e45862448412186","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.002468,"date_is_proxy":true,"latency_seconds":60.089,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.044},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":"high","source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-nano-2026-03-17"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"ad4b30120db2c934a6e00dae4b5bc8f3252d4adfd41d72c642148a2f4adff603","result":{"confidence_high":86.05524,"confidence_low":81.96276,"sample_count":null,"score":84.009,"score_display":"84.0","source_stated_rank":42},"run_fingerprint":"f67e65d891a4bc303104145d16f32593e043311c96f399287856e853d362a52d","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cdc8fad15ac7256796033b9cc2a0404a4908fb60805d7e965efd271e35b0fe7b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-5.4 nano (xhigh)","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_xhigh","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"f624d8dfc7b7e20fadb9b51d1762cbd0a925cf381e5a15eafdf1a91628dd06bd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.875,"score_display":"46.9","source_stated_rank":114},"run_fingerprint":"d9016869b6bd425bd91f6ec9911fdf0304b773dff3f01ff07e9345149eccce6d","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_0bdad69e8278c95ccb05d676","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6057c6347bab3f2f1e918f6afc0762ca147fc7694010a38f7d0c7310f65e7d2b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.027084","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gpt-5.4-nano-high","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"5540096dfcc8c3c297c5fa426d9e3ce6369532054938f8a51b1175afd64203a7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1004.52,"score_display":"1004.52","source_stated_rank":50},"run_fingerprint":"0465a531c087ba04b9dc115de4f68b371221070ccc17fa33c5a7bb2e2f01ec96","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"989bb8dc1ac1b445c239bb9680e85de766984ce9f7038d6b4aad4206c2af06eb","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.277298,"date_is_proxy":true,"latency_seconds":3276.224,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":5.075},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-nano-2026-03-17"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"7c95060fd4e1ed95d7a573ecc9e576508eb61faaf5d22a95fbe1df3f8fcf9697","result":{"confidence_high":36.044000000000004,"confidence_low":16.15,"sample_count":null,"score":26.097,"score_display":"26.1","source_stated_rank":68},"run_fingerprint":"3a3a797d92733d7def0a284745665108e245cd7f6c6afe03ac7ac17c9313a2d6","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"38cd54bb89a470e6f22871e176f04638024318ce3ff0b29848896141b3c46f74","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_high","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"50acdb50a06c35f2d73d3d2a986a6a213b3ef2d53afda42f0f791ddd3fadee20","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.23,"score_display":"49.2","source_stated_rank":73},"run_fingerprint":"419ca46297066dc79e517907137382375af3f267c42a7262eea4a973f9caeec5","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6574992399befa89380d64597313b8fc10ed79948b97e510004cbc703f48271e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-17","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"d6bd82a0dd80a7904066d7f6c3be1c1a501c1950f69cd511f4d04f06a53f3d04","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.96,"score_display":"38.0","source_stated_rank":126},"run_fingerprint":"b11f98e90d432366bc6ea461df87cfd5e9b2a9b206712f313ceb0b582f3d1196","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c34d174d18e7a8b7849cbd79","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7f07cdc3d838bf2cae4890ba313c273fc4e32318a742ed4733a3d6ccc19b321a","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.072321,"date_is_proxy":true,"latency_seconds":757.485,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.973},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":null,"upstream_model_id":"openai/gpt-5.4-nano-2026-03-17"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"ccd1e993725eb5c65c512763485a7356fb2d1d50e6e89e2a7b4ea5abaca54c4d","result":{"confidence_high":47.40008,"confidence_low":35.74592,"sample_count":null,"score":41.573,"score_display":"41.6","source_stated_rank":69},"run_fingerprint":"d1cd4bcc9a69d217130be8e44946026cd4643067045462f13cc600cf2f9d13fc","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5bec2cbd214ab2f1bec419a3","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"8fcbc669211f55bd776c57b131757b0f5bec1cda4e29b02e97777d9e45c06cac","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.255","mean_standard_error":"2.6","model_release_date":"2026-03-17","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.6,"upstream_source_url":null},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT 5.4 nano","source_effort":null,"source_model_version":"gpt-5.4-nano-2026-03-17_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"e945e248a857077699790c65240e59ad176d4f94f22a878083b0e1542a0d8527","result":{"confidence_high":21.996,"confidence_low":11.803999999999998,"sample_count":480,"score":16.9,"score_display":"16.9","source_stated_rank":45},"run_fingerprint":"6c6c44066e5b2e3360c3d2f85bf4596acee9aac84510c3a94c2140c7c44ca73b","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_18e983cf4d0d21d4c2fa2154","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d0125362f4c521bea12ac8ad5ac85ec93422390777d3807ea24b1dc95e814e01","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":400000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-03-17","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · GPT-5.4 nano (Xhigh) · gdpval"},"model":{"id":"openai/gpt-5-4-nano","name":"GPT 5.4 Nano","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"xhigh","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gpt-5-4-nano","upstream_model_label":"GPT-5.4 nano (Xhigh)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"ff1edbcb1f05a0189b68fafe8dd6595a41925812045cea0b038401992f4d347c","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":936.65,"score_display":"937","source_stated_rank":null},"run_fingerprint":"e1cf08cd91b222a7ce2f7ceaa02cd5dd3b5bfbda174c03c8647fe649e52d806b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · GPT-5.4 nano (Xhigh) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":5,"is_provider_default":false,"normalized_effort":"xhigh","profile_id":"profile_0bdad69e8278c95ccb05d676","provider_config":{"source_label":"xhigh"},"provider_mode_key":"XHigh","raw_label":"xhigh","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5f6f3675b4daba4a9375766fb00b877e67e16501b6f1e6482f570ec312c7a3f9","metric_details":{"license":"Proprietary","variance":2.642351973503724},"model":{"id":"xai/grok-4-1","name":"Grok 4.1","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1468.39315248474,"confidence_low":1462.02118691798,"sample_count":66366,"score":1465.20716970136,"score_display":"1465","source_stated_rank":52},"run_fingerprint":"3fceb23fc9d0c1abe933815e9e48bca9cc30c9d495ace5f92df4ec5d34a99d6b","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_308a2315a1ffdf31f407f589","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6c889592898e513c8a75e66c17ba708deba8e4764dc142a71b86a3fe90735732","metric_details":{"license":"Proprietary","variance":2.6204970453036656},"model":{"id":"xai/grok-4-1","name":"Grok 4.1","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1462.2603910365585,"confidence_low":1455.9148314901404,"sample_count":68478,"score":1459.0876112633496,"score_display":"1459","source_stated_rank":60},"run_fingerprint":"7a02c2730ad6519bc62bd4cd438863ef0697efac37a04357e878394462ec0bdf","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8037a088ae72e394cc8d79c3","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"943edf13e34b6d72176f02469e6fff40fde326afed7914733fde143e54da42e3","metric_details":{"license":"Proprietary","variance":160.84603346006048},"model":{"id":"xai/grok-4-1","name":"Grok 4.1","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"grok-4.1-thinking","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8e33826a45ab25399d691b1c5ff61f5e4cde6715830cf790ae0b76ce0c277d69","result":{"confidence_high":1238.6628684209863,"confidence_low":1188.9483468399105,"sample_count":961,"score":1213.8056076304483,"score_display":"1214","source_stated_rank":132},"run_fingerprint":"ee1fd34b26029a3a0233e31d0d4378aa56869584bc5e30402b443e88f9b3485b","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_308a2315a1ffdf31f407f589","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0e6e741bc85f80424538e9a3d3a141dab3b365a988fb1f9f6dac6dbd22bc8290","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-11-17","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"xai/grok-4-1","name":"Grok 4.1","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"grok-4-1","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"51136a17b8e19cb82ef5cdbb660cba27b1ac8d1664a6762bbce738f82418a9c3","result":{"confidence_high":1236.05,"confidence_low":1185.84,"sample_count":939,"score":1210.95,"score_display":"1210.95","source_stated_rank":128},"run_fingerprint":"97f5f85cd554aee7f5fce221f0280ad24231135e10582c00203304b41ee61b65","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1cf6171a00353fb18c151e92","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"522e7dcd1a4ae1260003ea288ad53a4013d584ef46e55db19cbe2b999f62ba52","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.24800000000000003","mean_standard_error":"2.6","model_release_date":"2025-11-17","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.3,"upstream_source_url":null},"model":{"id":"xai/grok-4-1","name":"Grok 4.1","provider":"xAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Grok 4.1","source_effort":null,"source_model_version":"grok-4-1","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"96b718460a7ea126b39a9a6c276b89750c4b4184626f0f2b8832c4820db5915f","result":{"confidence_high":17.308,"confidence_low":8.292000000000002,"sample_count":480,"score":12.8,"score_display":"12.8","source_stated_rank":50},"run_fingerprint":"4de24695265131547bd59d1c7a456b81880eae8ba2597ab57afe723e4e3dc83b","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_758c10a53eb8307370c6a2df","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c59e4c51950e0a2922b9081b8f5e707d4975b297ce661020c2cfdc1f92dbf1ca","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-04-29"},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":false,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen3-32B; model release: 2025-04-29","source_accessibility":"Open weights (unrestricted)","source_model_name":"Qwen3-32B","source_model_release_date":"2025-04-29","source_model_versions":["Qwen3-32B_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6ed72eff26e691ae4eae3864c1b90cc7d0b11bc6984fa83fd4dd85d1158bb1d2","result":{"confidence_high":140.36,"confidence_low":135.05,"sample_count":null,"score":138.51,"score_display":"138.51","source_stated_rank":122},"run_fingerprint":"c6d2feb9e9083be7f3b8330957d5f0abe07b9730e627d67d51954d82212e9d55","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_cebf1706df3e3613a3a126de","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"918cf5c1cb189779a39c11009ec3ede1e3e3c2a76d0e7ab52817be1cb9c96f27","metric_details":{"license":"Apache 2.0","variance":23.285774958815484},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1356.3857298920143,"confidence_low":1337.4699855913275,"sample_count":3926,"score":1346.9278577416708,"score_display":"1347","source_stated_rank":221},"run_fingerprint":"acaf29f041aba81c25e633514a8109f8e506a516a8d3bb25d4369c71dc2a92a4","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3aa37c0df95a457752b0f86c","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":35,"at":"2026-08-30T22:45:08.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:45:08.000Z","first_observed_at":"2026-09-01T00:20:17Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"35642ccbbb19f742bc51cabb73adfad4c912be50390d97475121ca60594d4f6f","metric_details":{"best_score_across_scorers":"0.5410353535353535","model_release_date":"2025-04-29","stderr":2.81462342016025},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"EXpHB6kFqDjRfG9A5LsDf6","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEXpHB6kFqDjRfG9A5LsDf6.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/EXpHB6kFqDjRfG9A5LsDf6.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"9f415d65db681547f574fafbfd1adb7f8ea722f85b42ef092c6e4a25fd9cb7a9","result":{"confidence_high":59.62019725704944,"confidence_low":48.58687345002126,"sample_count":null,"score":54.10353535353535,"score_display":"54.1","source_stated_rank":225},"run_fingerprint":"2bfbe61c74d71d0843f46cbddb20d132b4af430e11a5d6c9b79246313c37149c","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_cebf1706df3e3613a3a126de","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T17:33:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T17:33:22.000Z","first_observed_at":"2026-09-01T00:20:17Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"03f5e5365cd08575199b4e3b93e7117e6ad2cfc57ecb564ce2ce7f15ba3cb62e","metric_details":{"best_score_across_scorers":"0.6571969696969697","model_release_date":"2025-04-29","stderr":2.7890331058144286},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Ffoxqm7YsZr3FimCGsGAJv","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFfoxqm7YsZr3FimCGsGAJv.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Ffoxqm7YsZr3FimCGsGAJv.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"5803ddfc34eb39ac69ff768256d1e2165873ca384155fbabe39f43af7d22b354","result":{"confidence_high":71.18620185709325,"confidence_low":60.253192082300686,"sample_count":null,"score":65.71969696969697,"score_display":"65.7","source_stated_rank":197},"run_fingerprint":"10e3ceb5dc8ea25d7c3b0012c995bbd498b233918570bbe754c766cd1468b628","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0cca2de94475a140c59eab27","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:45:19.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:45:19.000Z","first_observed_at":"2026-09-01T00:20:16Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"efea098706d26c003ec5cc3217c718f4e2f188ced47d19499f3e4c2c92f6f25c","metric_details":{"best_score_across_scorers":"0.23055555555555557","model_release_date":"2025-04-29","stderr":5.148605337836995},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CfmvbqzjpF6cCJQfzqdD4w","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCfmvbqzjpF6cCJQfzqdD4w.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/CfmvbqzjpF6cCJQfzqdD4w.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7b3ac10f47270801e3a35e6dace9a96bafc344848517ae97dda76973b235815b","result":{"confidence_high":33.14682201771607,"confidence_low":12.964289093395047,"sample_count":45,"score":23.055555555555557,"score_display":"23.1","source_stated_rank":230},"run_fingerprint":"fb839d53c3282d6a523eef3dccbb919af38f7a17a45944fd3d55fe09656f00c0","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_cebf1706df3e3613a3a126de","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:44:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:44:49.000Z","first_observed_at":"2026-09-01T00:20:16Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6e2203d2aaddba026f174b6a897d0ae180cd58ec95f1c7fa9e20ef40304d8516","metric_details":{"best_score_across_scorers":"0.6694444444444444","model_release_date":"2025-04-29","stderr":6.0401063235303285},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"BAi4jidcT3F6reBkSXDJKv","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBAi4jidcT3F6reBkSXDJKv.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/BAi4jidcT3F6reBkSXDJKv.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"0e9e1c3f0c2d1e2f50048352b41bfa7843a6da5d9263dad7a195fab095fd0bc9","result":{"confidence_high":78.78305283856389,"confidence_low":55.105836050324996,"sample_count":45,"score":66.94444444444444,"score_display":"66.9","source_stated_rank":148},"run_fingerprint":"9be6ed4bed49b9a2c6d3e88038d5bdfa28fe6b21c579a95b2957af05797865e1","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2607c2730c4065871cc020e9","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"94634219899a68cb49298d13c72b5ed8c5f5a5a626701eb627a9b4845e939264","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-04-29","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3 32B (Reasoning)","source_effort":null,"source_model_version":"Qwen3-32B","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"eac59e785a258c2e38d55b2af57e244a9a7b33dcf9cfaa45c3270de9367d03bc","result":{"confidence_high":5.661392,"confidence_low":0.013679,"sample_count":71,"score":0.28571428571428603,"score_display":"0.3","source_stated_rank":156},"run_fingerprint":"a8c25336aad36e8c5f1e9c3884315074478d0dcc6315e8135a7564d53fa0d65c","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5eeb27f9b68bb2dd1366ce55","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T17:34:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T17:34:22.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"197637ce270b782907739c696b6b1baecc7e8bea4968665d01a6dc34df722ca0","metric_details":{"best_score_across_scorers":"0.01","model_release_date":"2025-04-29","stderr":0.9999999999999999},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"NV5tAZdTHYzP6F6Ym4dUus","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNV5tAZdTHYzP6F6Ym4dUus.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/NV5tAZdTHYzP6F6Ym4dUus.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"addfd9d9f52f7f414cfdf0ac476ef57364a292f319ae525d7d31cd89110773a6","result":{"confidence_high":2.96,"confidence_low":0.0,"sample_count":100,"score":1.0,"score_display":"1.0","source_stated_rank":189},"run_fingerprint":"03deaa4089a824c8c225e0ad7448a24629ad4cca7b34da6f614bfb8e16369e0f","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_cebf1706df3e3613a3a126de","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T17:33:24.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T17:33:24.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c998a527182eaac1e8817e4198c01b2b130d5d9705665172703b543c0660d86d","metric_details":{"best_score_across_scorers":"0.05","model_release_date":"2025-04-29","stderr":2.1904291355759034},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"MYGuFuvkWEwLC7US5UR363","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMYGuFuvkWEwLC7US5UR363.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/MYGuFuvkWEwLC7US5UR363.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"447713568cf7477392f79bb43440b54227f662d72b81352c5442f2a63335a966","result":{"confidence_high":9.293241105728772,"confidence_low":0.7067588942712293,"sample_count":100,"score":5.0,"score_display":"5.0","source_stated_rank":161},"run_fingerprint":"3057b6d9184ee3dc744a06bb38d9ce8288dfb3f9c4b586a964680b3c7722d4f5","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b627c1f91e67288b155bd5fc","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"29d1fd7fb8e7206b191d953c6fd75429d589da6fe693c271b368e869b33dc6a8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-04-29","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3 32B (Reasoning)","source_effort":null,"source_model_version":"Qwen3-32B","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"a8a54d7bb5464445029ee47794e5942fd4724f3060920e9fe4a6a64e74e5cfd0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.4166666666667,"score_display":"35.4","source_stated_rank":173},"run_fingerprint":"b5db125adde801b0039b9f364fa97211ede55471155595a42b321b2cdaa1c22a","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7b558a43320ab043f3b5f3e3","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:03:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:03:18Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cd33793f9ebd5eb7f1920add92d5603dfea50457d9123964e7e5b97d8d22bee7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":169.87,"leaderboard_last_updated_at":"2026-04-12","license":"apache-2.0","model_link":"https://huggingface.co/Qwen/Qwen3-32B","total_cost_usd":153.08},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"FC"},"run_count":null,"scaffold":"FC","tools_allowed":null},"protocol_fingerprint":"6a9c99bccbca11748bdf70a2bdd6eb864726c94bf4a029f84ab95898fe96d192","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.71,"score_display":"48.71","source_stated_rank":29},"run_fingerprint":"3a2a5707980d70897715555f206b9c658905ff52e75014ffe7230cd45ed5e559","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85ab9abe2438a5cb1fc4cac5","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":38,"at":"2026-08-27T22:30:42Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:42Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6dca037871e1fd60f553e8541af5a7e25ebc6cf5e74cecef5ed341aee4cc2a82","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":167.54,"leaderboard_last_updated_at":"2026-04-12","license":"apache-2.0","model_link":"https://huggingface.co/Qwen/Qwen3-32B","total_cost_usd":199.47},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prompted-function-calling","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"Prompt"},"run_count":null,"scaffold":"Prompt","tools_allowed":null},"protocol_fingerprint":"564fbea687964bbbeb4c2a4d92f977e2c00894a357d9ecf0f2b928b7737d37ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.78,"score_display":"46.78","source_stated_rank":33},"run_fingerprint":"a8da1d9210869530666382e6712a86c38fdea11303a7564aa8e3810c66e2bdaa","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85ab9abe2438a5cb1fc4cac5","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prompted-function-calling","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fc72cbfc1e078060a0c7da0928128d2780581fe2a4d68952acd145c636cbcf20","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":32768,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2025-04-28","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Qwen3 32B (Reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-32b-instruct-reasoning","upstream_model_label":"Qwen3 32B (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"7776504e9b6d00d1626748bf5e36b2a39b4624b557642ccd43420c061da36291","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":19.44,"score_display":"19","source_stated_rank":null},"run_fingerprint":"07ed9d12ff21a88053182a557ceadf2aa40a9be744478519d2a8b6a51f5c6726","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3 32B (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d3418e26019f2a26b71ed3db","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifbench","metric":"inst_level_strict_acc","name":"IFBench","release_id":"release_4adf348ac2f76539ed7ccb10","split_or_window":"ifbench-test-300-prompts-344-instructions","unit":"percent","version":"swallow-evaluation-instruct"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-04T14:29:18Z","observed_at":"2026-10-04T14:29:18Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c87ee84344739cf66d96a8f54cfe116d10f02e915868c57822fde2fcabfa4af1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","instruction_count":344,"model_release_date":"2025-04-29","prompt_count":300},"model":{"id":"alibaba/qwen3-32b","name":"Qwen 3 32B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"rule-based instruction checkers (lighteval 0.11.0 with Swallow fixes)","run_config":{"decoding":"temperature 0.6, top-p 0.95","leaderboard_revision":"c58a13a852eba061d639af7caf4ddbdff33b36e3","protocol":"Swallow LLM Leaderboard: swallow-evaluation-instruct (lighteval), allenai/IFBench_test, instruction-level strict accuracy; the final answer without the thinking part","swallow_model_id":"Qwen/Qwen3-32B","swallow_reasoning":"on"},"run_count":null,"scaffold":"swallow-evaluation-instruct (lighteval), swallow|ifbench_singleturn","tools_allowed":"none"},"protocol_fingerprint":"ee7e2333dcebcd23fdc8d1d794d58c4e202e42da23d768519a4735d0c3f880ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":300,"score":29.4,"score_display":"29.4","source_stated_rank":null},"run_fingerprint":"0f220e63f7c9a937537f26f094eefde7fb7f7a57bc8630e1cca5b98681770f95","source":{"content_hash":"fffdc5bb4b3de214a8a4ce885cf286e13a4b1488d9a4a0791e5918d1918b74cb","fetched_at":"2026-10-04T14:29:18Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://swallow-llm.github.io/leaderboard/index-post.en.html","id":"swallow-llm-leaderboard","last_fetched_at":"2026-10-04T14:29:18Z","license":"Leaderboard content CC-BY-4.0 (Swallow LLM Team, README of swallow-llm/leaderboard)","locator":null,"name":"Swallow LLM Leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/swallow-llm/leaderboard/c58a13a852eba061d639af7caf4ddbdff33b36e3/swallow_leaderboard_post_en.js"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d3418e26019f2a26b71ed3db","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d9cb970060c27544d047fe956781fb872603fbba6b710ebe0b633588ea98a3ab","metric_details":{"license":"Apache 2.0","variance":4.814802370883313},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1413.024780123425,"confidence_low":1404.4234162693542,"sample_count":28612,"score":1408.7240981963896,"score_display":"1409","source_stated_rank":150},"run_fingerprint":"c15c703a07024b86192931d4a67245e67e51dbb405a01d052cdc3289434fe10d","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e7d27865cc833f004dcca283","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_2f3c6e21d0f3616fdb78f627","split_or_window":"full-multimodal-cot-no-tools","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"aa2dc90a9c9803881d44c1b50e12eb93581ecff4c046d127ebc0056bdc37de1e","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE w/ CoT; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE with chain-of-thought evaluation; the card separately reports tool-augmented HLE; task count, judge, thinking configuration and run count not disclosed","question_count":2500,"reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"8c6303cd5e89dc378d11fb4b70a0ff9c3cccb55a031133c69d9cbde9481f1a8c","source_published_at":"2026-02-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5193c99f2c93e1107d46b911094f08dc7f18c6366de909d53f5f747daef3985a","result":{"confidence_high":26.019872,"confidence_low":22.65899,"sample_count":2500,"score":24.3,"score_display":"24.3","source_stated_rank":null},"run_fingerprint":"8f70acf791c9f0e0ca3cce26e0c9aa8213dd72869af99935664bd2fc013beeff","source":{"content_hash":"8c7de2992fd65eac13dd9ef6208f13b537fe166cd07b2937daa7227aae7e4a94","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:23Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-27B","id":"qwen3-5-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE w/ CoT; Qwen3.5-27B column","name":"Qwen3.5-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5f4005af8cee78d3b3a8a61","provider_config":{"source_id":"qwen3-5-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_287aa4bcb31b5b1ba26e562a","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"4652038c7964545259714489ea404705de849cb2eaed4db6e74b3457a2148970","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE evaluation; task count, split, tools, judge, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"4cacf5cf831c2e7165c8e718c71edb9fcd7e427e4ed4aeb02385a7200d28ca81","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.3,"score_display":"24.3","source_stated_rank":null},"run_fingerprint":"5ff911b1d38b818f4be22968c321424c125f2113b4d9d2fde95bad88d5053f52","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.5-27B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acca955d8107d22c457bcb69","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":222,"at":"2026-02-25","basis":"evaluation_at","evaluated_at":"2026-02-25","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4ab8f7ed0c0469dfd7e43c4cbec0e3fbda84ea6b72b6dc27e1560d242921cdbb","metric_details":{"modality_lane":"not reported","notes":"chain of thought","num_parameters":27781427952,"pull_request":11,"result_file_url":"https://huggingface.co/Qwen/Qwen3.5-27B/resolve/refs%2Fpr%2F11/.eval_results/hle_chain_of_thought_2.yaml","source":{"author":{"_id":"63e0eea7af523c37e5a77966","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1678663263366-63e0eea7af523c37e5a77966.jpeg","followerCount":485,"fullname":"Nathan Habib","isHf":true,"isHfAdmin":false,"isMod":false,"isPro":true,"isUserFollowing":false,"name":"SaylorTwift","type":"user"},"isExternal":false,"name":"Model Card","url":"https://huggingface.co/Qwen/Qwen3.5-27B"}},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.3,"score_display":"24.3","source_stated_rank":53},"run_fingerprint":"ce69b4dde399fe7611c88c419bdf5b783bd3f82d3c0ab428553785adb2b67b17","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d76925748fb8ff4791f5bb1a","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":222,"at":"2026-02-25","basis":"evaluation_at","evaluated_at":"2026-02-25","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b8d25357f7d83301c9279e2b9e029d3e428b1299990c339db71be29308a4f59d","metric_details":{"modality_lane":"not reported","notes":"with tools","num_parameters":27781427952,"pull_request":11,"result_file_url":"https://huggingface.co/Qwen/Qwen3.5-27B/resolve/refs%2Fpr%2F11/.eval_results/hle_with_tools_3.yaml","source":{"author":{"_id":"63e0eea7af523c37e5a77966","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1678663263366-63e0eea7af523c37e5a77966.jpeg","followerCount":485,"fullname":"Nathan Habib","isHf":true,"isHfAdmin":false,"isMod":false,"isPro":true,"isUserFollowing":false,"name":"SaylorTwift","type":"user"},"isExternal":false,"name":"Model Card","url":"https://huggingface.co/Qwen/Qwen3.5-27B"}},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.5,"score_display":"48.5","source_stated_rank":15},"run_fingerprint":"b5337a50d240c0b27a88264c20ada6985295d28ae04ee35b1661ba2b7c843a3f","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d76925748fb8ff4791f5bb1a","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_beac9b3474dc330fed0ebc77","split_or_window":"diamond-vendor-protocol-undisclosed","unit":"percent","version":"GPQA Diamond · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"90735b1194831c86757ca422ccf5424507f9e5630c1b91d28279b2b73b685146","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; STEM & Reasoning; GPQA Diamond; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card GPQA Diamond evaluation; prompt, shots, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"8c6303cd5e89dc378d11fb4b70a0ff9c3cccb55a031133c69d9cbde9481f1a8c","source_published_at":"2026-02-24","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"1ff9ae2ef8d730817a6d761b03392d3f8a0e65ea332abf727e3e7f33f3bf7ac6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.5,"score_display":"85.5","source_stated_rank":null},"run_fingerprint":"2a4db942edad015be6422bd3339808797367937820040ec2457a325b9b3d6352","source":{"content_hash":"8c7de2992fd65eac13dd9ef6208f13b537fe166cd07b2937daa7227aae7e4a94","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:23Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-27B","id":"qwen3-5-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; STEM & Reasoning; GPQA Diamond; Qwen3.5-27B column","name":"Qwen3.5-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5f4005af8cee78d3b3a8a61","provider_config":{"source_id":"qwen3-5-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_95dc04609d027072fa229d3b","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMLU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"4b1a0029e4c8a18e268217ed3357c4445e5b77badd12b3419dacf9f5775144bd","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMLU-Pro evaluation; prompt, shots, task count, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"70c7bbff02e177f65267ba3c33be7b6f38096fbaac03c9cc970c7210519303fa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.1,"score_display":"86.1","source_stated_rank":null},"run_fingerprint":"95819c8f70ac3e6071dec87e6d9ced7a537bbe2f8ed5ec3275fddf3faefe5799","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.5-27B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acca955d8107d22c457bcb69","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_95dc04609d027072fa229d3b","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMLU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"3bfae1c1fb700b6c6fa9219e6cec58d0013019f818af218f350b871c637d4ddb","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; Knowledge; MMLU-Pro; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMLU-Pro evaluation; prompt, shots, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"8c6303cd5e89dc378d11fb4b70a0ff9c3cccb55a031133c69d9cbde9481f1a8c","source_published_at":"2026-02-24","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"b3c7a0725963c573ade3c868fe396b7a93a503f6fc188b7571de6b1d0a16f448","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.1,"score_display":"86.1","source_stated_rank":null},"run_fingerprint":"d95842b2d93cbd8dd533f43160fee771a638ab6b6e5772473e0bf73a5d0ae3ea","source":{"content_hash":"8c7de2992fd65eac13dd9ef6208f13b537fe166cd07b2937daa7227aae7e4a94","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:23Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-27B","id":"qwen3-5-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; Knowledge; MMLU-Pro; Qwen3.5-27B column","name":"Qwen3.5-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5f4005af8cee78d3b3a8a61","provider_config":{"source_id":"qwen3-5-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_1dbdadbee92bb7f366d1025b","split_or_window":"standard","unit":"percent","version":"MMMU-Pro official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":160,"at":"2026-04-28","basis":"evaluation_at","evaluated_at":"2026-04-28","first_observed_at":"2026-08-29T23:52:50Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"420e99c3b27164aa4c01dfc5052fd9017ab458585ae0b8af37092a99e1d81df4","metric_details":{"num_parameters":27781427952,"pull_request":46,"result_file_url":"https://huggingface.co/Qwen/Qwen3.5-27B/resolve/refs%2Fpr%2F46/.eval_results/mmmu_pro.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/Qwen/Qwen3.5-27B"}},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9225c08e521da63df44b0ad31230f50489ffbee930c8fb4aaf2f0f8dbf6fce3f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.0,"score_display":"75.0","source_stated_rank":7},"run_fingerprint":"40ffcfa40610a2d3c06c8021c6649c6fd20ee4c78a813b3e7e122fde92f85f09","source":{"content_hash":"bc590a866070c9674cba0de4af7d92af6e0921bd2db419fca54624e00d73400b","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-10-05T12:33:31Z","homepage_url":"https://mmmu-benchmark.github.io/","id":"mmmu-pro","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"MMMU-Pro","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/api/datasets/MMMU/MMMU_Pro/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acc078689edb6302f6ab810f","provider_config":{"source_id":"mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_79984934735c6e815678be85","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMMU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"4b263459d101cf8edb328e181e66bd65036c335a23b696f41142b31a9f929e66","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMMU-Pro evaluation; prompt, task count, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"88ff021a3a75ecf4d3dcf90fa813de6de1b99cc0fac3e1a9502d28e8f43a47f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.0,"score_display":"75.0","source_stated_rank":null},"run_fingerprint":"48f1c09a29b67ce48fbbd9a1cd81014fb6a0b8af37ef0d759e705d306f1a33a1","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.5-27B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acca955d8107d22c457bcb69","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_d28d13464cb7d9d7990aac7d","split_or_window":"standard-vendor-protocol-undisclosed","unit":"percent","version":"MMMU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"b41abd06856cdee9e5f98e248f46d04b83d4c6d50ea794cfcc3a87690ad4903c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-27B official revision-pinned model card; Vision-Language benchmark table; STEM and Puzzle; MMMU-Pro; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMMU-Pro evaluation; prompt, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"8c6303cd5e89dc378d11fb4b70a0ff9c3cccb55a031133c69d9cbde9481f1a8c","source_published_at":"2026-02-24","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e0b708d5a82386a21eff567126140031c1b2ab7aaaea48d5c2cae8a3049831a9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.0,"score_display":"75.0","source_stated_rank":null},"run_fingerprint":"630dd51c306735b695f5e54e6722a7bb257a53425d55032b774f68085f137dfe","source":{"content_hash":"8c7de2992fd65eac13dd9ef6208f13b537fe166cd07b2937daa7227aae7e4a94","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:23Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-27B","id":"qwen3-5-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-27B official revision-pinned model card; Vision-Language benchmark table; STEM and Puzzle; MMMU-Pro; Qwen3.5-27B column","name":"Qwen3.5-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5f4005af8cee78d3b3a8a61","provider_config":{"source_id":"qwen3-5-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_9162beb27813d52a46a20a9f","split_or_window":"qwen-internal-agent-run","unit":"percent","version":"SWE-bench Verified · Qwen vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"508366e715fd26527334c243cb1c9d1856db291f10619131db2dd0ff3c3ae8b8","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.5-27B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Verified; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"dad3fccd5f7c7052561a2997a748ad0d5aa39550e8421c973d8656bf054626e7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.0,"score_display":"75.0","source_stated_rank":null},"run_fingerprint":"bc9c93323c34b402a644ae0f3990f4a838e51c6cbf88e2f8be9773f05b0ae477","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.5-27B column; SWE-Bench Series methodology note","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acca955d8107d22c457bcb69","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_73b4dac2493679abaf562468","split_or_window":"qwen-vendor-protocol-undisclosed","unit":"percent","version":"SWE-bench Verified · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"1d76841181531aa6ba886cae1ce4236a2ddf47d17dd5c1672a0c96bdd70bf79f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; Coding; SWE-bench Verified; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card SWE-bench Verified evaluation; task count, scaffold, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"8c6303cd5e89dc378d11fb4b70a0ff9c3cccb55a031133c69d9cbde9481f1a8c","source_published_at":"2026-02-24","tools":"coding-agent tools"},"run_count":null,"scaffold":"Qwen vendor agent scaffold not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"9db349fc670359df28eae55726e3b57fb1788ef5c0128f62b72098e876c80c91","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.4,"score_display":"72.4","source_stated_rank":null},"run_fingerprint":"aff20d8c5334cdf82ef8ddf25043e38c1eae2edaa71501a55550d7e456883e22","source":{"content_hash":"8c7de2992fd65eac13dd9ef6208f13b537fe166cd07b2937daa7227aae7e4a94","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:23Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-27B","id":"qwen3-5-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; Coding; SWE-bench Verified; Qwen3.5-27B column","name":"Qwen3.5-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5f4005af8cee78d3b3a8a61","provider_config":{"source_id":"qwen3-5-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_7449132b5f078c71f23c26b5","split_or_window":"2026-02-15/2026-03-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":218,"at":"2026-03-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5581ce3391010b25d03a445cf11f0cf35ff41327c8188d317f59bb10db49ad59","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-02-15","pass_at_5":59.25925925925925,"potential_contamination":false,"sem":1.385799032138497},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"Qwen3.5-27B__tools","window_from":"2026-02-15","window_status":"historical-post-release","window_to":"2026-03-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"8fe98e6ef0f26358cd9945f72ca92647ad82c74b572ec206946223c4d5374ded","result":{"confidence_high":57.53098091780627,"confidence_low":52.098648711823365,"sample_count":null,"score":54.81481481481482,"score_display":"54.81","source_stated_rank":null},"run_fingerprint":"7382af3cd1986618c6ac7bf6028f899e6656b4886399aad60256fe182f2457a3","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a806723737ab8f1e6f695e6f","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_d4a5abde3ed9874198f1dc1f","split_or_window":"qwen-refined-vendor-run","unit":"percent","version":"SWE-bench Pro · Qwen refined vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"1aca2bc947abdc59d59fe4a0b27e77ac1ee4b228bc4283f3abd877fb43e8c498","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.5-27B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Pro with Qwen's documented corrections/refinements to problematic tasks; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"5ea251f2329b35827abdf02131617fdfb607ea4f7d8ed5e602b3e023e39868d3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.2,"score_display":"51.2","source_stated_rank":null},"run_fingerprint":"8025b7960bb16f2399523975ef74c01d5172aa2294546b482815dfcfbf6a118b","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.5-27B column; SWE-Bench Series methodology note","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acca955d8107d22c457bcb69","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_b530ca4da413885c54f52baa","split_or_window":"qwen-vendor-reported","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:23Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"0848261135ac11f532113ccb5d135ef04f91e250deb1ed16917089f7f9d62430","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Benchmark table; livecodebench; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"LiveCodeBench v6; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"8c6303cd5e89dc378d11fb4b70a0ff9c3cccb55a031133c69d9cbde9481f1a8c","source_published_at":"2026-02-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"95d681e3f443f8d5d265d19ae8a8ce03ce12f86cdcded5df0926c1f3f815adf1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.7,"score_display":"80.7","source_stated_rank":null},"run_fingerprint":"fd7def26cab509f4c29b7e65b1d40d9eafed6a385cda7450ced51f753bbd948a","source":{"content_hash":"8c7de2992fd65eac13dd9ef6208f13b537fe166cd07b2937daa7227aae7e4a94","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:23Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-27B","id":"qwen3-5-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Benchmark table; livecodebench; Qwen3.5-27B column","name":"Qwen3.5-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5f4005af8cee78d3b3a8a61","provider_config":{"source_id":"qwen3-5-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"be0cd765d8d88476c3340c9cd0ca2c12825f458ac33e9a8ea026bbf340971925","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.041945","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-24","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"qwen3.5-27b","source_effort":null,"source_model_version":"qwen3.5-27B","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"11a3f9a7201ffbf5ce021e153c092dbee9b3c852f72843e74bf3a6f6832cc58c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":349.45,"score_display":"349.45","source_stated_rank":105},"run_fingerprint":"f77b719f3cdaa40c11704054716698df5d2c429e71ecc61cc209c43bd43e53d2","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d51b4d24a4d78655be5fdb73","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4880bdc504b52e12ddc7c963bdb7de7b87f24a6138afbcfbde038bca2a6c650c","metric_details":{"license":"Apache 2.0","variance":16.708117247424696},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.5-27b","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ede220e8ef7e7501fc3f79c4d52b810ca74939b23ea14c235eee9beda7c8d70d","result":{"confidence_high":1365.3487303594356,"confidence_low":1349.325803738208,"sample_count":7794,"score":1357.3372670488216,"score_display":"1357","source_stated_rank":104},"run_fingerprint":"01b35c622032246470887669e86b1f73faf5d8b83a2607926f3da43fdfec3f08","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d8f6ad970b239a16860c4cb8","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1e61b065c69c83463c1c64cb3c50d3e33d043b7879a3f055c17ddf86e3dd356a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-24","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.5-27B","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"ffb187a180aa0c6fbd10493efa5ab332a75e757138590188fc34e7a210500533","result":{"confidence_high":1365.55,"confidence_low":1349.2,"sample_count":7427,"score":1357.38,"score_display":"1357.38","source_stated_rank":102},"run_fingerprint":"cd940b151d9ba99c7c04b3abe1cf0ca4f7cbaf3b76253e63b31ad382cb83dc37","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_01df35a99e3cfc2964ea6e34","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ba5fa9a80230ecd57102096767a5a1792147420e8996b6ba1b7f3628d355829e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-24","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.5-27B","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"d3567b16c7cd49d2fa47f9c8fb940d89aebd07f44bfc96ba13c8315e6ba5b8b9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.53,"score_display":"39.5","source_stated_rank":114},"run_fingerprint":"9087341639a41f59b4f16ed063b97bd1896d5853f1d5c0dd26b5d6cd4f826c46","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9393d6a5cff56c2e2f1c57a5","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_rate","name":"τ³-Banking · Official harness","release_id":"release_c6e5abafbb7bffc0975932de","split_or_window":"qwen-official-user-model-bm25","unit":"percent","version":"TAU3-Bench · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"529e6c579b42eab22051cd0fc44e159ee11b9b89abe1b206d8498e589d6bab93","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; General Agent; TAU3-Bench; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card TAU3-Bench evaluation using the official GPT-5.2-low user model and default BM25 retrieval; domain aggregation, task count, model reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"TAU3 domain tools"},"run_count":null,"scaffold":null,"tools_allowed":"TAU3 domain tools"},"protocol_fingerprint":"ac033ae84d9e8c3792065e58a44a15f3d80b846134d957b52b2b43ded2458ebb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.4,"score_display":"68.4","source_stated_rank":null},"run_fingerprint":"3d45b9e0852085f6c506238cac53c43f1ea267ab7296f6e5abe6b9880db62feb","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; General Agent; TAU3-Bench; Qwen3.5-27B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acca955d8107d22c457bcb69","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_5af9fe5a9d5e275323798c8b","split_or_window":"self-contained-78-no-api-subset","unit":"percent","version":"SkillsBench · Qwen 78-task self-contained subset"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"14efd825387128108cdc67e04c9ca2bf2bb4a9545f39549fef3d6f6ce0c51d6c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 78 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.5-27B column; SkillsBench methodology note"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen vendor evaluation through OpenCode on 78 self-contained tasks, excluding API-dependent tasks; average of five runs; exact tool policy and reasoning mode not disclosed","question_count":78,"reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":5,"scaffold":"OpenCode","tools_allowed":"not reported"},"protocol_fingerprint":"1ef4df8bbd26617f6a97c4748bab4ad0b89cb62c839ee783c7a5b25a8ac69428","result":{"confidence_high":37.970374,"confidence_low":18.570065,"sample_count":78,"score":27.2,"score_display":"27.2","source_stated_rank":null},"run_fingerprint":"9fc927ca1b0f86703221415ca3933b9017d9c039297831285860f5a1b4913174","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.5-27B column; SkillsBench methodology note","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acca955d8107d22c457bcb69","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_6b9456463dae72f95fadf6b3","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"BFCL V4 · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"81b4ebffb3d2731b02c754f4c014f149f13ca4ef67fc13f096dc738a4530ad27","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; General Agent; BFCL-V4; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card BFCL-V4 evaluation; category mix, prompt, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"8c6303cd5e89dc378d11fb4b70a0ff9c3cccb55a031133c69d9cbde9481f1a8c","source_published_at":"2026-02-24","tools":"function-calling tools"},"run_count":null,"scaffold":null,"tools_allowed":"function-calling tools"},"protocol_fingerprint":"9aca4b26ad4c1fb4b38873d2a78f94206d521bb77b8fd1c6904a2c439362580d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.5,"score_display":"68.5","source_stated_rank":null},"run_fingerprint":"7c59d77b1784d000e4aa71ca70d868f07aa757e7e037d317225c8ab8b7fbd3b1","source":{"content_hash":"8c7de2992fd65eac13dd9ef6208f13b537fe166cd07b2937daa7227aae7e4a94","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:23Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-27B","id":"qwen3-5-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-27B official revision-pinned model card; Language benchmark table; General Agent; BFCL-V4; Qwen3.5-27B column","name":"Qwen3.5-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5f4005af8cee78d3b3a8a61","provider_config":{"source_id":"qwen3-5-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifeval","metric":"accuracy","name":"IFEval","release_id":"release_eaa2f0262bb0264860cb2c8e","split_or_window":"qwen-vendor-reported","unit":"percent","version":"IFEval"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:23Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"0f60a9e8e68f08820766291b50b9518c5af029b98299a55e1948489249a8077c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Benchmark table; ifeval; Qwen3.5-27B column"},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["ifeval_protocol_not_uniform"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"IFEval; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"8c6303cd5e89dc378d11fb4b70a0ff9c3cccb55a031133c69d9cbde9481f1a8c","source_published_at":"2026-02-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"e9e17b8e0074098f32daf83a5b38ae12858d3337cf27da604f333e45a9c74efb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":95.0,"score_display":"95.0","source_stated_rank":null},"run_fingerprint":"ad0b1c0431eb2137759034f5a08551f176b3e89a51a66e8fda903018206a14eb","source":{"content_hash":"8c7de2992fd65eac13dd9ef6208f13b537fe166cd07b2937daa7227aae7e4a94","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:23Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-27B","id":"qwen3-5-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Benchmark table; ifeval; Qwen3.5-27B column","name":"Qwen3.5-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f5f4005af8cee78d3b3a8a61","provider_config":{"source_id":"qwen3-5-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1ce67f445d95706389a23764db5adb6caefe443c015005263f4695e55f6d661f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-24","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen 3.5 27B","source_effort":null,"source_model_version":"qwen3.5-27B","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"6c1f16e150ac8721678431423fc4af65a8198ef9c08fa3300f1696b29819e7c9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":201.97599999999994,"score_display":"201.98","source_stated_rank":56},"run_fingerprint":"811ea4628f21c2fbc7f760562dfff6b55501ecf3b5ea21f6f550a9594eebba07","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7f48d5054d4b2dc26d1818ca","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifbench","metric":"inst_level_strict_acc","name":"IFBench","release_id":"release_4adf348ac2f76539ed7ccb10","split_or_window":"ifbench-test-300-prompts-344-instructions","unit":"percent","version":"swallow-evaluation-instruct"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-04T14:29:18Z","observed_at":"2026-10-04T14:29:18Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"364a58e2266ed75054278deaf8f86b1c610d9ae7cea42a038eb22903f5e917ca","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","instruction_count":344,"model_release_date":"2026-02-16","prompt_count":300},"model":{"id":"alibaba/qwen3-5-27b","name":"Qwen 3.5 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"rule-based instruction checkers (lighteval 0.11.0 with Swallow fixes)","run_config":{"decoding":"temperature 0.6, top-p 0.95","leaderboard_revision":"c58a13a852eba061d639af7caf4ddbdff33b36e3","protocol":"Swallow LLM Leaderboard: swallow-evaluation-instruct (lighteval), allenai/IFBench_test, instruction-level strict accuracy; the final answer without the thinking part","swallow_model_id":"Qwen/Qwen3.5-27B","swallow_reasoning":"on"},"run_count":null,"scaffold":"swallow-evaluation-instruct (lighteval), swallow|ifbench_singleturn","tools_allowed":"none"},"protocol_fingerprint":"9b012eb36f1c2f4cf17e351131da2df24841c3192da533ea9c19d25ddab91dc8","result":{"confidence_high":null,"confidence_low":null,"sample_count":300,"score":64.5,"score_display":"64.5","source_stated_rank":null},"run_fingerprint":"0787a798a6bd1c7f7632699f20bd8d9b6332cdcf8cf1433473fa15196b9d1d9e","source":{"content_hash":"fffdc5bb4b3de214a8a4ce885cf286e13a4b1488d9a4a0791e5918d1918b74cb","fetched_at":"2026-10-04T14:29:18Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://swallow-llm.github.io/leaderboard/index-post.en.html","id":"swallow-llm-leaderboard","last_fetched_at":"2026-10-04T14:29:18Z","license":"Leaderboard content CC-BY-4.0 (Swallow LLM Team, README of swallow-llm/leaderboard)","locator":null,"name":"Swallow LLM Leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/swallow-llm/leaderboard/c58a13a852eba061d639af7caf4ddbdff33b36e3/swallow_leaderboard_post_en.js"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_de190b968e73c379f7673b4a","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1a517d0b875053889b59b38a111ee5c55e1f8018194fd37ce34008b713d53706","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-24"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen3.5-35B-A3B; model release: 2026-02-24","source_accessibility":"Open weights (unrestricted)","source_model_name":"Qwen3.5-35B-A3B","source_model_release_date":"2026-02-24","source_model_versions":["qwen3.5-35b-a3b","qwen3.5-35b-a3b_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6f340822467ccbaca8b874c83d73b0510b998f9344532706c3ef8f1b4c421f7e","result":{"confidence_high":144.75,"confidence_low":140.16,"sample_count":null,"score":142.52,"score_display":"142.52","source_stated_rank":97},"run_fingerprint":"e7b05538dfd6792c0405672a5f2ffb42588877647704ba4e18f57d15519a49a6","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8038866fb4362de4cdb57daf","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"28e94e3971d7de70d740a2e273e762be763edea3a33e7b7cf0f897f5db73eb7e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-02-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"qwen3-5-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.5 35B A3B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"74f86b1e3b96a7b2e3eb9692773cc9cf2b672e6e4f790b26311c88c84db65c47","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":15.115982,"score_display":"15.1","source_stated_rank":null},"run_fingerprint":"35cbd797734b13330d3d4b4c3011da8d49e0d30f1b038ef03d69a9b5b29327ba","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fb6e6b6eb588dd82385e85cf5281820564c4edb6c7197468da6423c18b343c97","metric_details":{"license":"Apache 2.0","variance":4.680716887881697},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1398.0337692063858,"confidence_low":1389.5530189621857,"sample_count":30408,"score":1393.7933940842859,"score_display":"1394","source_stated_rank":168},"run_fingerprint":"60cc95e335f84203565c3ffcbd46579865a77adaa2e5470c34966b0028f12c4a","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7bb891886f6b9f82e595d537","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_2f3c6e21d0f3616fdb78f627","split_or_window":"full-multimodal-cot-no-tools","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"171536dbc27ea4f24a2de988e75a17ccf2194b216ef1ecf4d6a9b649af6e1373","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE w/ CoT; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE with chain-of-thought evaluation; the card separately reports tool-augmented HLE; task count, judge, thinking configuration and run count not disclosed","question_count":2500,"reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"71c3bbc2eadadd5980bbdbb15df958c27d42ede24821015602a021109aacf492","source_published_at":"2026-02-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b49926e35945bdfc4bf990d42caf170ca771797b88a16f10251ffb64499db7de","result":{"confidence_high":24.075975,"confidence_low":20.808717,"sample_count":2500,"score":22.4,"score_display":"22.4","source_stated_rank":null},"run_fingerprint":"cc316e55ab5d1340ddbd43bdbaeaf712dc551414a11ee5b85c4af871634ee7d6","source":{"content_hash":"53f95493fda35b0d088d4483d145bc749ca3bc44c20a7e7d9cbe854c89b74931","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","id":"qwen3-5-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE w/ CoT; Qwen3.5-35B-A3B column","name":"Qwen3.5-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acf8ff0a882897edb172285e","provider_config":{"source_id":"qwen3-5-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_287aa4bcb31b5b1ba26e562a","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"1f48235ce8e00e229f922f4ba2211806fb525c0e663788a60d1327828a5e8e6b","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE evaluation; task count, split, tools, judge, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"4cacf5cf831c2e7165c8e718c71edb9fcd7e427e4ed4aeb02385a7200d28ca81","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":22.4,"score_display":"22.4","source_stated_rank":null},"run_fingerprint":"2ea02597117a0ef5ce069389d60f2ab9cf572924b758422a32fc0c355decb298","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.5-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a489b097f55632aa33d8a051","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"77da94c8654823e552e054ee2bc4e333d81a7bea8e91a4023d52aa1de5add2b7","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-02-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · hle"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"qwen3-5-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.5 35B A3B (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c82f11f93ed09ee059ff29283f4a431b1a4d7e73b955c89638baeca630283a3f","result":{"confidence_high":14.942546,"confidence_low":12.064133,"sample_count":2158,"score":13.438369,"score_display":"13.4","source_stated_rank":null},"run_fingerprint":"3ca6d672d608f8242ce99f58ef7c4e7c28db27d74857a7e2d2839b51e39d6909","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"565c94de520aeb07fb3b4c77e629cdc80ad06ca82afc2c6012670a034cdb1d52","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-02-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"qwen3-5-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.5 35B A3B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b73e5cd99b7c054fc9be875f4120b9f11e2bae8cc6e3ec141337a49314ca1bfd","result":{"confidence_high":86.655811,"confidence_low":75.967554,"sample_count":198,"score":81.919192,"score_display":"81.9","source_stated_rank":null},"run_fingerprint":"ab8c732c2cb942be11260bf0c77f94cdbb39b3ca9ead81c3688a43075fdee3c8","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T18:43:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T18:43:36.000Z","first_observed_at":"2026-09-01T00:20:17Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0061bd7e85294b99f35c3085c5d171c9a300f4c9fbe30dcb51a353e5e029f1f6","metric_details":{"best_score_across_scorers":"0.8118686868686869","model_release_date":"2026-02-24","stderr":2.319862922976808},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"KKm5DbPUgXo88eYr64g4SK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKKm5DbPUgXo88eYr64g4SK.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/KKm5DbPUgXo88eYr64g4SK.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"2b6512c1b2dce3e8e4b3a28b661df9c525062c0513c14bd426f23278bdf4de7f","result":{"confidence_high":85.73380001590323,"confidence_low":76.63993735783413,"sample_count":null,"score":81.18686868686868,"score_display":"81.2","source_stated_rank":123},"run_fingerprint":"d7fc3790bffd84f929a0009a107a4d041352133fdc1ff4d066cd2a74b5742a7b","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T18:38:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T18:38:38.000Z","first_observed_at":"2026-09-01T00:20:17Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fadd4ff210f5e4b2cab7543bb3081f6acf140a66f1352c326e61d6bc3af7986b","metric_details":{"best_score_across_scorers":"0.8345959595959596","model_release_date":"2026-02-24","stderr":2.179018174848238},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"RWjuCuVGvaRd4vDjJJ6aP2","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRWjuCuVGvaRd4vDjJJ6aP2.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/RWjuCuVGvaRd4vDjJJ6aP2.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"a5eb67475fa88bf3c8a5cc8efed7de1c2a95202d4a047cb04541ed1ab7441ce9","result":{"confidence_high":87.7304715822985,"confidence_low":79.18872033689341,"sample_count":null,"score":83.45959595959596,"score_display":"83.5","source_stated_rank":102},"run_fingerprint":"40d1e5242c445ebad462d6506054c955b492ecf818e7f55cae499e3a8a6ac71e","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_078bfd52287e19c1de1bedaf","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_beac9b3474dc330fed0ebc77","split_or_window":"diamond-vendor-protocol-undisclosed","unit":"percent","version":"GPQA Diamond · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"ad6024288dac52e4f4010e36640c1fad049fd71d7cdf1bdc1b76f73f70499f65","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; GPQA Diamond; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card GPQA Diamond evaluation; prompt, shots, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"71c3bbc2eadadd5980bbdbb15df958c27d42ede24821015602a021109aacf492","source_published_at":"2026-02-24","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"a3b39e0cd6f03946393684d963ca2cb007da0bd8cdbc7fb358ee1c5b42c20532","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.2,"score_display":"84.2","source_stated_rank":null},"run_fingerprint":"baf4214cba03397f517189377ac8aaef987cdc2686026b9b64444cda83c21658","source":{"content_hash":"53f95493fda35b0d088d4483d145bc749ca3bc44c20a7e7d9cbe854c89b74931","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","id":"qwen3-5-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; GPQA Diamond; Qwen3.5-35B-A3B column","name":"Qwen3.5-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acf8ff0a882897edb172285e","provider_config":{"source_id":"qwen3-5-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_95dc04609d027072fa229d3b","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMLU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"b4c0ba1cdf51a27be1e0b529016fcd72f3682b596b5c039e60a3167eb288b08f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMLU-Pro evaluation; prompt, shots, task count, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"70c7bbff02e177f65267ba3c33be7b6f38096fbaac03c9cc970c7210519303fa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.3,"score_display":"85.3","source_stated_rank":null},"run_fingerprint":"0f5d4241e89874f8a37a8a560aebce700d8b24e146f66a4887a677bea0b699ef","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.5-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a489b097f55632aa33d8a051","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_95dc04609d027072fa229d3b","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMLU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"277b1fa1bd60f3499e24fc0f292bc4ceee2dd3a356f9109a8ec4e94c10fcda0c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; Knowledge; MMLU-Pro; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMLU-Pro evaluation; prompt, shots, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"71c3bbc2eadadd5980bbdbb15df958c27d42ede24821015602a021109aacf492","source_published_at":"2026-02-24","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"f11c3bee18a5fe6535d00462de9f56dea193bc4c47184acb8ecd690a9734acf7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.3,"score_display":"85.3","source_stated_rank":null},"run_fingerprint":"38ee0d3af3dd08301aa9da58123084f118c321363649b7092495be6cfb9b47ab","source":{"content_hash":"53f95493fda35b0d088d4483d145bc749ca3bc44c20a7e7d9cbe854c89b74931","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","id":"qwen3-5-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; Knowledge; MMLU-Pro; Qwen3.5-35B-A3B column","name":"Qwen3.5-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acf8ff0a882897edb172285e","provider_config":{"source_id":"qwen3-5-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:45:56.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:45:56.000Z","first_observed_at":"2026-09-01T00:20:16Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cb58b9f6cfda5b3a89afbf6dabb5422e513615a0ed99d4f740e7c5f60a6213b1","metric_details":{"best_score_across_scorers":"0.7","model_release_date":"2026-02-24","stderr":6.003996985174109},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"h9aoDxyWEhybvARHpyuoco","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fh9aoDxyWEhybvARHpyuoco.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/h9aoDxyWEhybvARHpyuoco.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"91930fa35f725ad3c356dbd7a26c502c4a1cd60c9d733a134b49eb2cf966042e","result":{"confidence_high":81.76783409094125,"confidence_low":58.23216590905875,"sample_count":45,"score":70.0,"score_display":"70.0","source_stated_rank":139},"run_fingerprint":"cfb74c9ea31d186c4532af18d05a890834f78956e43dd6d7235f3275ae87ce09","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:45:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:45:36.000Z","first_observed_at":"2026-09-01T00:20:16Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b51dba6110343393de6a757339e7e91b5cb8d8e09171d0ef388c58944f70dbea","metric_details":{"best_score_across_scorers":"0.5444444444444444","model_release_date":"2026-02-24","stderr":6.194903049563445},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"9WS6kHXT8XFtJWtE2wWLAo","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9WS6kHXT8XFtJWtE2wWLAo.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/9WS6kHXT8XFtJWtE2wWLAo.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9cfe49a50a0c9965eaed1ab4ff625bef3bfd586fd77c141769b0803191759c6f","result":{"confidence_high":66.5864544215888,"confidence_low":42.30243446730009,"sample_count":45,"score":54.44444444444444,"score_display":"54.4","source_stated_rank":179},"run_fingerprint":"6b42d65e07f0934346c22a348ab8e7c08b4d1d31652d16a0703ccf56405450cd","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_924bd364b1bffa2c058d3e77","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"46772251eac6fa7a9cf61d5ae60c0a80acdc15afe6e5aa1d0343894cf5e89a63","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-02-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"qwen3-5-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.5 35B A3B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f5aae5ac6ff3c3614cd499cc0dc6857e5b4b834398f1722f1b9ecd67fa0f1ef0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.190751,"score_display":"69.2","source_stated_rank":null},"run_fingerprint":"55170197b63105bc97e67f4d77dc0a3f977f968eeb7a3b247fd2ea181ca9beeb","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_79984934735c6e815678be85","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMMU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"80415a3a60b3433cbb65a83ec01e8fa430c592ccbe030bcf3430af465d4cfbac","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMMU-Pro evaluation; prompt, task count, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"88ff021a3a75ecf4d3dcf90fa813de6de1b99cc0fac3e1a9502d28e8f43a47f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.1,"score_display":"75.1","source_stated_rank":null},"run_fingerprint":"62d23a60cfefa657b27b152732301196ad9bc189d740ad4781a1d058773955cb","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.5-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a489b097f55632aa33d8a051","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_d28d13464cb7d9d7990aac7d","split_or_window":"standard-vendor-protocol-undisclosed","unit":"percent","version":"MMMU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"53b00f079369d4f57679c5f5714ab2996d3dc03183ea8fd6c5587861f205002a","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-35B-A3B official revision-pinned model card; Vision-Language benchmark table; STEM and Puzzle; MMMU-Pro; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMMU-Pro evaluation; prompt, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"71c3bbc2eadadd5980bbdbb15df958c27d42ede24821015602a021109aacf492","source_published_at":"2026-02-24","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"2baca78a9e174d4d7886ac3c5c71784db84905215bdf6e65d31169f48375b5b2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.1,"score_display":"75.1","source_stated_rank":null},"run_fingerprint":"183d743f8dc6964f90608ddc059207c68fa79fa5b01d2b60c1e21f04cb43cc9e","source":{"content_hash":"53f95493fda35b0d088d4483d145bc749ca3bc44c20a7e7d9cbe854c89b74931","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","id":"qwen3-5-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-35B-A3B official revision-pinned model card; Vision-Language benchmark table; STEM and Puzzle; MMMU-Pro; Qwen3.5-35B-A3B column","name":"Qwen3.5-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acf8ff0a882897edb172285e","provider_config":{"source_id":"qwen3-5-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"350ec50703b1910a4f6bcd1f1692eca305f04072c73f96044d55d3e5764ff999","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-24","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.5 35B A3B (Non-reasoning)","source_effort":null,"source_model_version":"qwen3.5-35b-a3b_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"5883adcd5901ca67361c21b62e568dafd4443eeaad296150e4b6a0027d4276ec","result":{"confidence_high":6.166938,"confidence_low":0.050231,"sample_count":71,"score":0.571428571428571,"score_display":"0.6","source_stated_rank":153},"run_fingerprint":"b56f31a6dd72977c8cab560024860d8d7aebf28d7eb48e9338e209b4a5e6e410","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c7dbab4883793f4c59ed4acc27709dd4134a74e92304faa1a394ded76baafee2","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-02-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · critpt"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"qwen3-5-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.5 35B A3B (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"8eb54bfc4d1f9137886139a42f014faaf8b19ebea8dd5ef25cf4e0be857fc0cf","result":{"confidence_high":6.236252,"confidence_low":0.049636,"sample_count":70,"score":0.571429,"score_display":"0.6","source_stated_rank":null},"run_fingerprint":"f2ced06fd161d73d7ae2b5c7715d85ad2bf5d085c48f4a2debb9f48b98b92d4b","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:45:52.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:45:52.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"43ffdfe9cb3da51f4e1489fdafebe2751a5acb9d47da6b31c605d17098401cac","metric_details":{"best_score_across_scorers":"0.1","model_release_date":"2026-02-24","stderr":3.015113445777636},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jzqDbWHvQXhMkpYHfTjnRE","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjzqDbWHvQXhMkpYHfTjnRE.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/jzqDbWHvQXhMkpYHfTjnRE.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"6dd81b025488ad00ce576832875d43d89e910fd78b5e3d909563ffbc8801b2cc","result":{"confidence_high":15.909622353724167,"confidence_low":4.090377646275833,"sample_count":100,"score":10.0,"score_display":"10.0","source_stated_rank":137},"run_fingerprint":"60649b86b51e55d17d9305561f98f020a146b226a559b276b05a75a9900e4d04","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T18:43:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T18:43:15.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a1140424d742fbd0bb5f655ada180910829ce288c1d7ae1ac96589b8ccd8ae96","metric_details":{"best_score_across_scorers":"0.01","model_release_date":"2026-02-24","stderr":0.9999999999999999},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"KxxB8kDa9obqrPuec4PEiv","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKxxB8kDa9obqrPuec4PEiv.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/KxxB8kDa9obqrPuec4PEiv.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"408c4c4a2f9622be62ec8e30b9f044b9e24551a75de39f7ceb6007505cea96b4","result":{"confidence_high":2.96,"confidence_low":0.0,"sample_count":100,"score":1.0,"score_display":"1.0","source_stated_rank":189},"run_fingerprint":"bedf5d3089378d036109bd38417a3dd014ab5076695f9ed385f53bce0132bbf0","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_26a2015b4c28af79c1344730","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_9162beb27813d52a46a20a9f","split_or_window":"qwen-internal-agent-run","unit":"percent","version":"SWE-bench Verified · Qwen vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"31a81390ecd810045cdffee48ed7e139c4c8244a463f2acba0952d929c8f2fb8","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.5-35B-A3B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Verified; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"dad3fccd5f7c7052561a2997a748ad0d5aa39550e8421c973d8656bf054626e7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.0,"score_display":"70.0","source_stated_rank":null},"run_fingerprint":"b91ad4b09950831548895f062638ccf94225cc8fcb4af4646b49154793c68906","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.5-35B-A3B column; SWE-Bench Series methodology note","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a489b097f55632aa33d8a051","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_73b4dac2493679abaf562468","split_or_window":"qwen-vendor-protocol-undisclosed","unit":"percent","version":"SWE-bench Verified · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"4ecbd9a00ca2b3cb6354f2299889328daa724f5c0d1e2d1273d6308b52b9494e","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; Coding; SWE-bench Verified; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card SWE-bench Verified evaluation; task count, scaffold, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"71c3bbc2eadadd5980bbdbb15df958c27d42ede24821015602a021109aacf492","source_published_at":"2026-02-24","tools":"coding-agent tools"},"run_count":null,"scaffold":"Qwen vendor agent scaffold not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"55c51f2feeba256dfd6b63116ffec6aad4749136bdfdacf9c3805ed8f4d7acab","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.2,"score_display":"69.2","source_stated_rank":null},"run_fingerprint":"a825e85d2f360baa81d12efef3c376b1ea6559c4b52dffba5e20f2148c49956e","source":{"content_hash":"53f95493fda35b0d088d4483d145bc749ca3bc44c20a7e7d9cbe854c89b74931","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","id":"qwen3-5-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; Coding; SWE-bench Verified; Qwen3.5-35B-A3B column","name":"Qwen3.5-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acf8ff0a882897edb172285e","provider_config":{"source_id":"qwen3-5-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"a00999461d5a15b2c4520d00b039f525824db79177232cf2feab5f7fc9d7b05c","metric_details":{"model_release_date":"2026-02-15","pass_at_5":36.93693693693694,"potential_contamination":false,"sem":1.06596032127921},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"Qwen3.5-35B-A3B__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"6e5a0da2f64d15a75563cdfb237ee200ce210a778081a94c786af397623a1536","result":{"confidence_high":19.206399346824366,"confidence_low":15.027834887409863,"sample_count":null,"score":17.117117117117115,"score_display":"17.12","source_stated_rank":13},"run_fingerprint":"d222bdb3496ac12279d1b3ab91a8ffa1a6b2ed382df032656260aa96da7694cb","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_be91adc28b2c36400b8ba1d2","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_d4a5abde3ed9874198f1dc1f","split_or_window":"qwen-refined-vendor-run","unit":"percent","version":"SWE-bench Pro · Qwen refined vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"fb7cdeb17594812506f678aa060a071358853484f82bdfbbb7ea7fea5335a14e","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.5-35B-A3B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Pro with Qwen's documented corrections/refinements to problematic tasks; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"5ea251f2329b35827abdf02131617fdfb607ea4f7d8ed5e602b3e023e39868d3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.6,"score_display":"44.6","source_stated_rank":null},"run_fingerprint":"fe0d9b7f714de80ab06fb17a6788265e4fdf13d42b89bc9711f6b478c0f36e08","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.5-35B-A3B column; SWE-Bench Series methodology note","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a489b097f55632aa33d8a051","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_b530ca4da413885c54f52baa","split_or_window":"qwen-vendor-reported","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:24Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"58cf9274dd79179aadedb6d26afe7a4de1155417f03ad69f5b7a8715badf18d0","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Benchmark table; livecodebench; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"LiveCodeBench v6; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"71c3bbc2eadadd5980bbdbb15df958c27d42ede24821015602a021109aacf492","source_published_at":"2026-02-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"db4e6883454c44a81391e2be412d53f1f000436285ebff88bf47b93dfaaf4dae","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.6,"score_display":"74.6","source_stated_rank":null},"run_fingerprint":"cc1229feb272d3d3078a3cf81e8c6c7dcfe54db5f95098084c9c40be82424d2d","source":{"content_hash":"53f95493fda35b0d088d4483d145bc749ca3bc44c20a7e7d9cbe854c89b74931","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","id":"qwen3-5-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Benchmark table; livecodebench; Qwen3.5-35B-A3B column","name":"Qwen3.5-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acf8ff0a882897edb172285e","provider_config":{"source_id":"qwen3-5-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9a0385ba5d8b30c6536f2222468dfcb9cb5a9f574dce3797133a0651eaf4faa7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.5 35B A3B (Non-reasoning)","source_effort":null,"source_model_version":"qwen3.5-35b-a3b_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"1a198886d4fc358ff800abfc9a5c92e2bc8dee83167d73c119f59429dd9b55bf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":29.2824074074074,"score_display":"29.3","source_stated_rank":183},"run_fingerprint":"ab9caa0d99d44274c0c1e39d7263c891596bc12f09bfdc2bba8ba866b20e155b","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"93640eba707d685f9e830c4551be278770610c1644f3cf108553bbaecdf2606c","metric_details":{"license":"Apache 2.0","variance":78.84829652630496},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.5-35b-a3b","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3ca1852f7a805b3a4d871fb3d6adb783bf80fc15fc3ea54cb582816fb150098c","result":{"confidence_high":1270.797480374088,"confidence_low":1235.989867173084,"sample_count":1617,"score":1253.3936737735858,"score_display":"1253","source_stated_rank":125},"run_fingerprint":"bb64cdcd016bb0aacde879fba35cade9d6bdb78ab426e217d62ed462a99b95ed","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_629f0be1378c8068b93135c5","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"734b4bb72a6ebc3e592a13d500b9e3c3eae8facb10248c3089fa9bf1fdec90f8","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-02-24","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-5-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.5 35B A3B (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"f2b361ff5b17541ab507cb160241fcafa9a6674485d1d00f853a48c327a53570","result":{"confidence_high":51.208912,"confidence_low":31.198399,"sample_count":89,"score":40.82397,"score_display":"40.8","source_stated_rank":null},"run_fingerprint":"e44eac0e9b0094abcec8e3cf11840e92cd0c5297f012db7c5b1e65179e7a2b6e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_rate","name":"τ³-Banking · Official harness","release_id":"release_c6e5abafbb7bffc0975932de","split_or_window":"qwen-official-user-model-bm25","unit":"percent","version":"TAU3-Bench · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"a67f709ca26801d56ab6e79690408d0807c344540de4f486e3ee4ca2d155f896","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; General Agent; TAU3-Bench; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card TAU3-Bench evaluation using the official GPT-5.2-low user model and default BM25 retrieval; domain aggregation, task count, model reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"TAU3 domain tools"},"run_count":null,"scaffold":null,"tools_allowed":"TAU3 domain tools"},"protocol_fingerprint":"ac033ae84d9e8c3792065e58a44a15f3d80b846134d957b52b2b43ded2458ebb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":68.9,"score_display":"68.9","source_stated_rank":null},"run_fingerprint":"ae5081bd124fd991bdcb417a116f31652e02bb443d052ed2c1a9267a1559fa4a","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; General Agent; TAU3-Bench; Qwen3.5-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a489b097f55632aa33d8a051","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-17T14:33:50Z","observed_at":"2026-09-17T14:33:50Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"652132c1663d8cc4d7672f49bcf4fa5379fce691de1bc334fabe84351f79925d","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-02-24","source_listing":{"last_listed_at":"2026-09-17T14:33:50Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · tauBanking"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"3ea78499e45c73ef3ba8f20f216d42eb460ab1aab73e4c87f420d14b8e237850","upstream_model_id":"qwen3-5-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.5 35B A3B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"86784d5fdedb2bd1b029547fa4a787eeca76d6b99e30a3c187610a4b91db16e2","result":{"confidence_high":11.232424,"confidence_low":2.096996,"sample_count":97,"score":4.948454,"score_display":"4.9","source_stated_rank":null},"run_fingerprint":"ee503dfe9dc12847528d3e6c09025b18de8b273bc5aa31d6b72eff9373ef3a47","source":{"content_hash":"6c2a6eff6f29b24250e2c7e2f75c15c428796328b899ce3ee28cc0e9d2e1e4e0","fetched_at":"2026-09-17T14:33:50Z","first_fetched_at":"2026-09-17T14:33:50Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-17T14:33:50Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_5af9fe5a9d5e275323798c8b","split_or_window":"self-contained-78-no-api-subset","unit":"percent","version":"SkillsBench · Qwen 78-task self-contained subset"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"8d3661f56a87d13740247d4a672dd945c65110093cf50964ec5fd737a16e6507","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 78 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.5-35B-A3B column; SkillsBench methodology note"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen vendor evaluation through OpenCode on 78 self-contained tasks, excluding API-dependent tasks; average of five runs; exact tool policy and reasoning mode not disclosed","question_count":78,"reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":5,"scaffold":"OpenCode","tools_allowed":"not reported"},"protocol_fingerprint":"1ef4df8bbd26617f6a97c4748bab4ad0b89cb62c839ee783c7a5b25a8ac69428","result":{"confidence_high":11.472588,"confidence_low":1.60829,"sample_count":78,"score":4.4,"score_display":"4.4","source_stated_rank":null},"run_fingerprint":"320dbfb4604193a6eaeef0ad21a11f9eecc81e9dc1b7e148e0f22154b166cfc8","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.5-35B-A3B column; SkillsBench methodology note","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a489b097f55632aa33d8a051","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_6b9456463dae72f95fadf6b3","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"BFCL V4 · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"ea0af1f6215972d71f3a4b2fd3d790adf4829ea0426ba4cf7d4ce7ee2354ec7c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; General Agent; BFCL-V4; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card BFCL-V4 evaluation; category mix, prompt, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"71c3bbc2eadadd5980bbdbb15df958c27d42ede24821015602a021109aacf492","source_published_at":"2026-02-24","tools":"function-calling tools"},"run_count":null,"scaffold":null,"tools_allowed":"function-calling tools"},"protocol_fingerprint":"d2e5e4ded552b5be2636ae22be91934e3af5094a2c843dba75092617f97b0e8a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.3,"score_display":"67.3","source_stated_rank":null},"run_fingerprint":"ec02988d19fe94097e752e7d934900ce8829321a4996eb59f285dad0733ce83f","source":{"content_hash":"53f95493fda35b0d088d4483d145bc749ca3bc44c20a7e7d9cbe854c89b74931","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","id":"qwen3-5-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-35B-A3B official revision-pinned model card; Language benchmark table; General Agent; BFCL-V4; Qwen3.5-35B-A3B column","name":"Qwen3.5-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acf8ff0a882897edb172285e","provider_config":{"source_id":"qwen3-5-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifeval","metric":"accuracy","name":"IFEval","release_id":"release_eaa2f0262bb0264860cb2c8e","split_or_window":"qwen-vendor-reported","unit":"percent","version":"IFEval"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":223,"at":"2026-02-24","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:24Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-24","status":"stale"},"measurement_id":"9acd3fb07b5b7882ec632c448c9b55d601504cc1242ba9ea16104334b021827f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Benchmark table; ifeval; Qwen3.5-35B-A3B column"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["ifeval_protocol_not_uniform"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"IFEval; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-02-24","source_content_hash_method":"raw_bytes","source_content_sha256":"71c3bbc2eadadd5980bbdbb15df958c27d42ede24821015602a021109aacf492","source_published_at":"2026-02-24","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"75c65dff8a4ca072f7779ee663a82f005dc599d80742e995c92e9efe54e12762","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.9,"score_display":"91.9","source_stated_rank":null},"run_fingerprint":"f93a5f4172e3a754eb1c169b80f3846b83e4860f8578764e449d162e948239ef","source":{"content_hash":"53f95493fda35b0d088d4483d145bc749ca3bc44c20a7e7d9cbe854c89b74931","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","id":"qwen3-5-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Benchmark table; ifeval; Qwen3.5-35B-A3B column","name":"Qwen3.5-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_acf8ff0a882897edb172285e","provider_config":{"source_id":"qwen3-5-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0c744c3c83d12923c695c9a5d515550f7ca584a157bc1970317e1021eb2c3f61","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-02-24","reported_confidence_interval":"-29 / +29","source_locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-5-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.5 35B A3B (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"cf59b9f5ce0e8c73be65071ef44dd9b337a5a38607e555a542dfab0063b4a47d","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":594.66,"score_display":"595","source_stated_rank":null},"run_fingerprint":"7253a04035b649bf55f2a467369663a1aa4c8e87f7c5c988bd82eaf7c80cc128","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 35B A3B (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_08198280c56f559e82197c5d","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifbench","metric":"inst_level_strict_acc","name":"IFBench","release_id":"release_4adf348ac2f76539ed7ccb10","split_or_window":"ifbench-test-300-prompts-344-instructions","unit":"percent","version":"swallow-evaluation-instruct"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-04T14:29:18Z","observed_at":"2026-10-04T14:29:18Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"eca7d6e060781481802685d019969481f5e9a6ded322c4f48bd9c379a2cb490a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","instruction_count":344,"model_release_date":"2026-02-16","prompt_count":300},"model":{"id":"alibaba/qwen3-5-35b-a3b","name":"Qwen 3.5 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"rule-based instruction checkers (lighteval 0.11.0 with Swallow fixes)","run_config":{"decoding":"temperature 0.6, top-p 0.95","leaderboard_revision":"c58a13a852eba061d639af7caf4ddbdff33b36e3","protocol":"Swallow LLM Leaderboard: swallow-evaluation-instruct (lighteval), allenai/IFBench_test, instruction-level strict accuracy; the final answer without the thinking part","swallow_model_id":"Qwen/Qwen3.5-35B-A3B","swallow_reasoning":"on"},"run_count":null,"scaffold":"swallow-evaluation-instruct (lighteval), swallow|ifbench_singleturn","tools_allowed":"none"},"protocol_fingerprint":"637455e9814ebdb49cbf54e163208050c724c6ab8f9e5de8f6d8bcd522d407e0","result":{"confidence_high":null,"confidence_low":null,"sample_count":300,"score":59.9,"score_display":"59.9","source_stated_rank":null},"run_fingerprint":"877ddbf051b5bcab60fa8f7770661a20dea8bfc844c555f5c0071e33cf73b5e2","source":{"content_hash":"fffdc5bb4b3de214a8a4ce885cf286e13a4b1488d9a4a0791e5918d1918b74cb","fetched_at":"2026-10-04T14:29:18Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://swallow-llm.github.io/leaderboard/index-post.en.html","id":"swallow-llm-leaderboard","last_fetched_at":"2026-10-04T14:29:18Z","license":"Leaderboard content CC-BY-4.0 (Swallow LLM Team, README of swallow-llm/leaderboard)","locator":null,"name":"Swallow LLM Leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/swallow-llm/leaderboard/c58a13a852eba061d639af7caf4ddbdff33b36e3/swallow_leaderboard_post_en.js"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_f3e955ec7b7b358b2d54c536","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"af23f71e02f9c80ff9750d6bd5ebc0390cc00159804b1fd5fa49382ecb5ed8f9","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-22"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen3.6 27B; model release: 2026-04-22","source_accessibility":"Open weights (unrestricted)","source_model_name":"Qwen3.6 27B","source_model_release_date":"2026-04-22","source_model_versions":["qwen3.6-27b","qwen3.6-27b_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"dc76178a814e52616db4b0cf3437195da8f3760251375d09ef24ae6d311032f0","result":{"confidence_high":147.92,"confidence_low":144.24,"sample_count":null,"score":146.47,"score_display":"146.47","source_stated_rank":70},"run_fingerprint":"1acde23ce787d04379bd2972dd41c47a73e67351b0f5f4a8f027c39712454d99","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d79e65ecde0b2da0fa5b6c49","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a71e1266ca983830e6fb83e06706d874aa9cea528f05d2cc57ee9bffdd52d20f","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b","upstream_model_label":"Qwen3.6 27B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"19e5ef37de744a84a793152f0fcd2be29ea688660e96f9c156dd2fe675a79b34","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":21.905065,"score_display":"21.9","source_stated_rank":null},"run_fingerprint":"5d0301f258f18709c3d0ce937f8379bc96aa883facfecb30abc1483d76809167","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_532636b8035b61ccdae69abd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b0830d74eb0a9816b9fe9009de5ac35e084dbb6b4fec65b6c6452e98e8a4c267","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b-non-reasoning","upstream_model_label":"Qwen3.6 27B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"7cb0bb2694c4c4fba943aeb0a17102c604785dfb14e321f294e0106256c69c87","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":19.82882,"score_display":"19.8","source_stated_rank":null},"run_fingerprint":"d80b65fce4dd607dbfbfe442032d48040388819141f532185064581aa3da6203","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"23c23ae8c0b105ba3487b38b05e8b1f5c7496a85faddf747c8f4dd8d76bc3404","metric_details":{"category_scores":{"Agentic Coding":39.29266666666667,"Coding":71.785,"Data Analysis":70.42666666666666,"IF":53.229,"Language":63.30433333333334,"Mathematics":79.8685,"Reasoning":70.28375}},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.02713095238094,"score_display":"64.03","source_stated_rank":64},"run_fingerprint":"0144080ac492f0770b601c144cdb79983e3ab106a1f07e5264d7fff8040e74fd","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9e9091d6396e29d6d8b835ff","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"35029e9a2483a48b9ac103f298944535d2412e605f1a8505539c07c7933e68f0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":null,"num_parameters":27781427952,"pull_request":2,"result_file_url":"https://huggingface.co/Qwen/Qwen3.6-27B/resolve/refs%2Fpr%2F2/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/Qwen/Qwen3.6-27B"}},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.0,"score_display":"24","source_stated_rank":54},"run_fingerprint":"6cef0faca0e6192b842964c7cd6804cc8c76ae98fb46ffc1da1ad2ba85d092c2","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3570f866437a50d449060a12","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_5c8194fb4db887ad53fb49c9","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"HLE · Qwen3.8 card comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"1e2ce2dd67d18d1e1582c4f084d0fcb2b162a29fc1310f8d0bbe94b51dd478de","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Text Performance table; General; Multidisciplinary reasoning HLE; Qwen3.6-27B column; HLE methodology note"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":"GPT-4o","run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen3.8 model-card comparison on HLE judged by GPT-4o; task count, split, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"0783527ab425709637f1c9426d28ec004f4381a73b976c23ba9bd56b2f47bec7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.0,"score_display":"24.0","source_stated_rank":null},"run_fingerprint":"2d699d82f4f050ea42ce0b14a77af62de338642ecc3859513a899e6588f0acaa","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Text Performance table; General; Multidisciplinary reasoning HLE; Qwen3.6-27B column; HLE methodology note","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a26de80da0b74a12cbb6375e","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_287aa4bcb31b5b1ba26e562a","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"0a999f046dc6605dd7f24dc2cbb0299ef9a2f5e658e5f9f7696465ae793f173f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE evaluation; task count, split, tools, judge, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"a84a031c29362410cc334384b07675f01d0dae9a081b10ca8c1b04573c33fbea","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.0,"score_display":"24.0","source_stated_rank":null},"run_fingerprint":"e8deb70647ab503fc5968b95a03d8d3367af64ec77c4108d497cc30b3909f8b4","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.6-27B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_609752b0f7bd8a6341aa27a9","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aa1ecbe216fd23e7c674758051b3491c9acd8a341061cb5d65f9e62b389702ed","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · hle"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b","upstream_model_label":"Qwen3.6 27B (Reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"cf0c5313fdfacffc253255ba0019c8f3863debca16c6843efd7ca675067ba9c0","result":{"confidence_high":24.901485,"confidence_low":21.348046,"sample_count":2158,"score":23.076923,"score_display":"23.1","source_stated_rank":null},"run_fingerprint":"47f9dc12d4fcd60444216b4d8052beb99e53fd429a26ed7dad35d82f4131ee65","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_532636b8035b61ccdae69abd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"10dc257048036e300e326f822032ac8fb1db16b7fd66d6f73377905eac7a7cda","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · hle"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b-non-reasoning","upstream_model_label":"Qwen3.6 27B (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a17c9e9f0779b25a17c074cc634dcc4061f40fc0ec152accc69b2e7fe8408e26","result":{"confidence_high":16.631309,"confidence_low":13.613349,"sample_count":2158,"score":15.060241,"score_display":"15.1","source_stated_rank":null},"run_fingerprint":"d196ee2c126cf8211d898ba6069c8eda4aa772acc1409937c382dcf897742cfe","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"508ea427fea910bac2e0d6d8495e33a10db5cd4c2fcc8aeca6909b4e3a2f3eff","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b","upstream_model_label":"Qwen3.6 27B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fa2fbf8361f9daea6f1967f15ede01589b5889326a0d9e09a2f1c0752c0cbb64","result":{"confidence_high":88.659216,"confidence_low":78.522177,"sample_count":198,"score":84.242424,"score_display":"84.2","source_stated_rank":null},"run_fingerprint":"ec7728c28a745fb0284a471a4587a672bf81cf3c868472d90cdffc7a1fa66c72","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_532636b8035b61ccdae69abd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"10ff2a9d2154556e05af06276dd9d08a698df68a29ae26c4ff6b466b40e54cca","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b-non-reasoning","upstream_model_label":"Qwen3.6 27B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"15257c6e62575abda7e3a452259134c99b91c40d2ebe2253dbf85c7af16e3874","result":{"confidence_high":87.531012,"confidence_low":77.074105,"sample_count":198,"score":82.929293,"score_display":"82.9","source_stated_rank":null},"run_fingerprint":"018a5eac6458397819ed835be47a15dd2bbddc2e34e8fb6c1a52bcd824629016","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:24:27.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:24:27.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6a4772560f9700d825a10c3da756dff3be65603ae66ee1019be1df16b3be4baf","metric_details":{"best_score_across_scorers":"0.8585858585858586","model_release_date":"2026-04-22","stderr":2.4825909793343355},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"buLf8Ne29xmzMZBRQkvrQG","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbuLf8Ne29xmzMZBRQkvrQG.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/buLf8Ne29xmzMZBRQkvrQG.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"7f3bdcfd5893163cacbc94e7ad798db85e80b7d362802efb0096c9b9992cfb55","result":{"confidence_high":90.72446417808115,"confidence_low":80.99270753909056,"sample_count":null,"score":85.85858585858585,"score_display":"85.9","source_stated_rank":82},"run_fingerprint":"bcf392bbf5eed2931067434f6b7968bd96b92266fa22cf2e688a7a75f7dad3ea","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c365a945c629a18fa3c918d2","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:24:17.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:24:17.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d54aed8c862471ca815d4fd5634229b0fca1d155b24e1309924f342dde27aab7","metric_details":{"best_score_across_scorers":"0.8484848484848485","model_release_date":"2026-04-22","stderr":2.554565042660364},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"WeP7Q27N5PzxSgg4ybgijX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWeP7Q27N5PzxSgg4ybgijX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/WeP7Q27N5PzxSgg4ybgijX.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"902aa0213d71ed279670e7a8ae2a957fa4acf691bf2fda16e2ceeea0bd02acac","result":{"confidence_high":89.85543233209916,"confidence_low":79.84153736487053,"sample_count":null,"score":84.84848484848484,"score_display":"84.8","source_stated_rank":92},"run_fingerprint":"3d1f4045eedc69520fc6d7e335704ece4392999dea45c3d25cff4263e69e969a","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_213077da6e1af8428d045d9c","split_or_window":"diamond-vendor-protocol-undisclosed","unit":"percent","version":"GPQA Diamond · Qwen3.8 card comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"8121c973b35614df95dd9d33686bc6d512377143cc325e28cdae3b2f0e438db9","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Text Performance table; General; Scientific reasoning GPQA Diamond; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen3.8 model-card comparison on GPQA Diamond; prompt, shots, tool policy, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"610e60c6888f4b71966dec65719c1cfe6550665544de09fb14a70077c84538c9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.8,"score_display":"87.8","source_stated_rank":null},"run_fingerprint":"6eb0063309520d543073f4de2a2bd01078a3a8fc4ffb676a80475a00f53d8b65","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Text Performance table; General; Scientific reasoning GPQA Diamond; Qwen3.6-27B column","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a26de80da0b74a12cbb6375e","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_d7983b534cfcbf543e50af4a","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"GPQA Diamond · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"8fe4532d5e9e05e61d4b5007c8e89379c4ef32b920e94f01936764cf9b9de28c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; STEM & Reasoning; GPQA Diamond; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":6,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card GPQA Diamond evaluation; prompt, task count, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"dd2cbcaba371ecefe249fd5325da514fd694c6b98d78339e083b881ae780a231","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.8,"score_display":"87.8","source_stated_rank":null},"run_fingerprint":"b96e96221c0543ae50c4c1a06980d0fb1105584be52397f90bc6054125bcf1db","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; STEM & Reasoning; GPQA Diamond; Qwen3.6-27B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_609752b0f7bd8a6341aa27a9","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_ac2ec96fc175d2759bceb841","split_or_window":"demietrich-mmlu-pro-subset-140","unit":"percent","version":"MMLU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":87,"at":"2026-07-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-10","status":"fresh"},"measurement_id":"7f10fefa96c08854ab33dfd190395cad7fccce1d4401ff34b08bedc471ec5af4","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 140 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://demietrich.com/blog/qwen36-quant-shootout/","evidence_sha256":"daf77d9e65cabe65e8ce1fc168d110a690afaef00ba07145a60bfd5c23833445","kind":"reviewed-static-report","published_at":"2026-07-10","reviewed_at":"2026-09-05","source_url":"https://demietrich.com/blog/qwen36-quant-shootout/","version_id":"2716ecb401c028b19783edcd97a485ca56b471a3f34a014a46ccea63562eff89"},"source_locator":"Benchmark table; mmlu-pro; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Demietrich","evidence_verified":true,"freshness_proxy":true,"protocol":"MMLU-Pro; 140-sample subset; greedy temperature 0, thinking disabled as reported; llama.cpp; 2x RTX3090 host. Exact subset IDs and checkpoint revision not published.","quantization":{"backend":"llama.cpp","base_checkpoint":"Qwen/Qwen3.6-27B","hardware":"2 × RTX 3090 host; 1 GPU for Q4","id":"q4-k-xl","label":"Q4_K_XL","limitations":"Small subsets; backend and GPU count vary. Compare complete configurations, not quantization alone. Exact checkpoint revisions and per-question answers are undisclosed.","reference":false,"study_id":"demietrich-qwen36-quant","study_label":"Demietrich · 2026-07-10"},"question_count":140,"reasoning_effort":"off","result_published_at":"2026-07-10","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"78c3c033f6360ceef5650d3f5954e180b18c56492325613e3248e7cf69cb8c48","source_published_at":"2026-07-10","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8b5a7a5036a2081dbd6a8f3e8af81c269eb2e569544b9644bbdf0119c4cfb66f","result":{"confidence_high":78.863153,"confidence_low":64.156391,"sample_count":140,"score":72.1,"score_display":"72.1","source_stated_rank":null},"run_fingerprint":"2d6dafcbce2c22410890a68f05a3078b22c1c48faf15a6aac950858d5da02652","source":{"content_hash":"e3e3e61d0f0166b17263ff0c90a3945dee11a937b8942d7c7dc4047d272420bd","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://demietrich.com/blog/qwen36-quant-shootout/","id":"demietrich-qwen36-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; mmlu-pro; Qwen3.6-27B column","name":"Demietrich Qwen3.6-27B quantization study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://demietrich.com/blog/qwen36-quant-shootout/"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_ac2ec96fc175d2759bceb841","split_or_window":"demietrich-mmlu-pro-subset-140","unit":"percent","version":"MMLU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":87,"at":"2026-07-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-10","status":"fresh"},"measurement_id":"5b983cf91be423496c9af6ef32e7b7f8adf80bdfd26b388535901cb13d3f49c7","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 140 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://demietrich.com/blog/qwen36-quant-shootout/","evidence_sha256":"daf77d9e65cabe65e8ce1fc168d110a690afaef00ba07145a60bfd5c23833445","kind":"reviewed-static-report","published_at":"2026-07-10","reviewed_at":"2026-09-05","source_url":"https://demietrich.com/blog/qwen36-quant-shootout/","version_id":"2716ecb401c028b19783edcd97a485ca56b471a3f34a014a46ccea63562eff89"},"source_locator":"Benchmark table; mmlu-pro; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Demietrich","evidence_verified":true,"freshness_proxy":true,"protocol":"MMLU-Pro; 140-sample subset; greedy temperature 0, thinking disabled as reported; llama.cpp; 2x RTX3090 host. Exact subset IDs and checkpoint revision not published.","quantization":{"backend":"llama.cpp","base_checkpoint":"Qwen/Qwen3.6-27B","hardware":"2 × RTX 3090 host; 1 GPU for Q4","id":"q6-k-xl","label":"Q6_K_XL","limitations":"Small subsets; backend and GPU count vary. Compare complete configurations, not quantization alone. Exact checkpoint revisions and per-question answers are undisclosed.","reference":true,"study_id":"demietrich-qwen36-quant","study_label":"Demietrich · 2026-07-10"},"question_count":140,"reasoning_effort":"off","result_published_at":"2026-07-10","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"78c3c033f6360ceef5650d3f5954e180b18c56492325613e3248e7cf69cb8c48","source_published_at":"2026-07-10","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"132c2bb5bd914e31c2992472b6790d07731e43cac87df23ea0d8b27555d96c62","result":{"confidence_high":78.863153,"confidence_low":64.156391,"sample_count":140,"score":72.1,"score_display":"72.1","source_stated_rank":null},"run_fingerprint":"730a6ba865e3f297d5519a00d4c139c20b8b84eca6205c5c8a0901232e0823f5","source":{"content_hash":"e3e3e61d0f0166b17263ff0c90a3945dee11a937b8942d7c7dc4047d272420bd","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://demietrich.com/blog/qwen36-quant-shootout/","id":"demietrich-qwen36-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; mmlu-pro; Qwen3.6-27B column","name":"Demietrich Qwen3.6-27B quantization study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://demietrich.com/blog/qwen36-quant-shootout/"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_ac2ec96fc175d2759bceb841","split_or_window":"demietrich-mmlu-pro-subset-140","unit":"percent","version":"MMLU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":87,"at":"2026-07-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-10","status":"fresh"},"measurement_id":"225f2b514870b5b352d1f235452d35e5131929397d6a69b35e192bf5cdc068d6","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 140 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://demietrich.com/blog/qwen36-quant-shootout/","evidence_sha256":"daf77d9e65cabe65e8ce1fc168d110a690afaef00ba07145a60bfd5c23833445","kind":"reviewed-static-report","published_at":"2026-07-10","reviewed_at":"2026-09-05","source_url":"https://demietrich.com/blog/qwen36-quant-shootout/","version_id":"2716ecb401c028b19783edcd97a485ca56b471a3f34a014a46ccea63562eff89"},"source_locator":"Benchmark table; mmlu-pro; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Demietrich","evidence_verified":true,"freshness_proxy":true,"protocol":"MMLU-Pro; 140-sample subset; greedy temperature 0, thinking disabled as reported; vLLM; 2x RTX3090 host. Exact subset IDs and checkpoint revision not published.","quantization":{"backend":"vLLM","base_checkpoint":"Qwen/Qwen3.6-27B","hardware":"2 × RTX 3090 host; 1 GPU for Q4","id":"int4-autoround","label":"INT4-AutoRound","limitations":"Small subsets; backend and GPU count vary. Compare complete configurations, not quantization alone. Exact checkpoint revisions and per-question answers are undisclosed.","reference":false,"study_id":"demietrich-qwen36-quant","study_label":"Demietrich · 2026-07-10"},"question_count":140,"reasoning_effort":"off","result_published_at":"2026-07-10","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"78c3c033f6360ceef5650d3f5954e180b18c56492325613e3248e7cf69cb8c48","source_published_at":"2026-07-10","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"de4060aba73c8f6c12e6985662dd73858d732b7c629a6e2e3476fbe15ea2d025","result":{"confidence_high":76.973875,"confidence_low":61.957839,"sample_count":140,"score":70.0,"score_display":"70.0","source_stated_rank":null},"run_fingerprint":"f392694133012ec0decdc1f60134543a8bf6e0feb62754b23ba39300bf3e3543","source":{"content_hash":"e3e3e61d0f0166b17263ff0c90a3945dee11a937b8942d7c7dc4047d272420bd","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://demietrich.com/blog/qwen36-quant-shootout/","id":"demietrich-qwen36-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; mmlu-pro; Qwen3.6-27B column","name":"Demietrich Qwen3.6-27B quantization study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://demietrich.com/blog/qwen36-quant-shootout/"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_ac2ec96fc175d2759bceb841","split_or_window":"demietrich-mmlu-pro-subset-140","unit":"percent","version":"MMLU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":87,"at":"2026-07-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-10","status":"fresh"},"measurement_id":"c3ccba119e93fae2cf510906cc0b65e618851d8245d114073cedd763a12975d8","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 140 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://demietrich.com/blog/qwen36-quant-shootout/","evidence_sha256":"daf77d9e65cabe65e8ce1fc168d110a690afaef00ba07145a60bfd5c23833445","kind":"reviewed-static-report","published_at":"2026-07-10","reviewed_at":"2026-09-05","source_url":"https://demietrich.com/blog/qwen36-quant-shootout/","version_id":"2716ecb401c028b19783edcd97a485ca56b471a3f34a014a46ccea63562eff89"},"source_locator":"Benchmark table; mmlu-pro; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Demietrich","evidence_verified":true,"freshness_proxy":true,"protocol":"MMLU-Pro; 140-sample subset; greedy temperature 0, thinking disabled as reported; vLLM; 2x RTX3090 host. Exact subset IDs and checkpoint revision not published.","quantization":{"backend":"vLLM","base_checkpoint":"Qwen/Qwen3.6-27B","hardware":"2 × RTX 3090 host; 1 GPU for Q4","id":"fp8","label":"FP8","limitations":"Small subsets; backend and GPU count vary. Compare complete configurations, not quantization alone. Exact checkpoint revisions and per-question answers are undisclosed.","reference":false,"study_id":"demietrich-qwen36-quant","study_label":"Demietrich · 2026-07-10"},"question_count":140,"reasoning_effort":"off","result_published_at":"2026-07-10","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"78c3c033f6360ceef5650d3f5954e180b18c56492325613e3248e7cf69cb8c48","source_published_at":"2026-07-10","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c2ab2da21554053df766856123e60bab831c1a517e2fe91f27ad40645b6cec29","result":{"confidence_high":75.066495,"confidence_low":59.777389,"sample_count":140,"score":67.9,"score_display":"67.9","source_stated_rank":null},"run_fingerprint":"036cc543fbd7fea47caf2f5ac0f413278a0b4567c5bee7fb59404fd4cd977e8a","source":{"content_hash":"e3e3e61d0f0166b17263ff0c90a3945dee11a937b8942d7c7dc4047d272420bd","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://demietrich.com/blog/qwen36-quant-shootout/","id":"demietrich-qwen36-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; mmlu-pro; Qwen3.6-27B column","name":"Demietrich Qwen3.6-27B quantization study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://demietrich.com/blog/qwen36-quant-shootout/"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_95dc04609d027072fa229d3b","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMLU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"0a3a26d152b241aa7292684e9295d46dc64da81b6368356d472be114ac9db40f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMLU-Pro evaluation; prompt, shots, task count, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"44f56864a490bb7582461036c068400061930b80aee3b634127a43235a3bd532","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":86.2,"score_display":"86.2","source_stated_rank":null},"run_fingerprint":"b91072c0a84c1a6fcf0c4add3474cce51465210c5cfc7de6f30fd732769ddd19","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.6-27B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_609752b0f7bd8a6341aa27a9","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:12:36.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:12:36.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e3533c3461e62d73f057a0529a1128052fe330f1092f3099cd5f04cc3134cc9a","metric_details":{"best_score_across_scorers":"0.9111111111111111","model_release_date":"2026-04-22","stderr":4.290254662948546},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"452pB5w5ea6uy92e2q2NMv","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F452pB5w5ea6uy92e2q2NMv.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/452pB5w5ea6uy92e2q2NMv.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"62c42b96674f98c0a3d367bb21907b85a8fd1fa83ec55da20c0692d255928e09","result":{"confidence_high":99.52001025049026,"confidence_low":82.70221197173197,"sample_count":45,"score":91.11111111111111,"score_display":"91.1","source_stated_rank":65},"run_fingerprint":"208713f11f946b70be635ee518120f5c1d21e2c90148cd01e732c6c23bb58800","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_de27cbb86e9c990231514038","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:12:23.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:12:23.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a284d7337b59f741859585a07c99e15c678d401beb93650bcc2b5dddb015578f","metric_details":{"best_score_across_scorers":"0.6666666666666666","model_release_date":"2026-04-22","stderr":7.106690545187015},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"o6VtA8qruLpoShhPy2JdLg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fo6VtA8qruLpoShhPy2JdLg.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/o6VtA8qruLpoShhPy2JdLg.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b58acf5001cbfa04d66bf38ff5a078d4d4a76d0ee2cbd4d39417d86323cd7b4c","result":{"confidence_high":80.59578013523321,"confidence_low":52.73755319810011,"sample_count":45,"score":66.66666666666666,"score_display":"66.7","source_stated_rank":149},"run_fingerprint":"85feed12d66f98357beb5855aba9d72dc52a28a1f9203fa54fe3a710ec502b92","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-29T11:06:28.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-29T11:06:28.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"252afd206c6d9ad024674a5ee1891efb0a3969ae528d2fa52a5009b7eb0e32e8","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.34035087719298246","leaderboard_private_problem_count":285,"model_release_date":"2026-04-22","public_example_count":10,"stderr":2.8116467881852296},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"WjXdTPMnJUAT27azUKo3Ka","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWjXdTPMnJUAT27azUKo3Ka.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/WjXdTPMnJUAT27azUKo3Ka.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.6 27B","thinking":null,"upstream_model_id":"qwen3.6-27b_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"1bed8ed4070c75c3320000f0d35a74716111e019ba019ae7fde7b7195d434cab","result":{"confidence_high":39.5459154241413,"confidence_low":28.524260014455198,"sample_count":285,"score":34.03508771929825,"score_display":"34.0","source_stated_rank":65},"run_fingerprint":"8efb5c97cbe5dd42fc9d6ea0d8143c5493921129450536aebb1887edc0c9cd36","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-28T12:30:22.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T12:30:22.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f75c6b9c6c05711a38832a0384177787bc523c30787d8a666c0de1fe7b7b95b0","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.3508771929824561","leaderboard_private_problem_count":285,"model_release_date":"2026-04-22","public_example_count":10,"stderr":2.8319255656547093},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"VonoEhAsGSJ2FHU33qnhui","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVonoEhAsGSJ2FHU33qnhui.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/VonoEhAsGSJ2FHU33qnhui.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.6 27B","thinking":null,"upstream_model_id":"qwen3.6-27b","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"aaad37169a9e137984ff69df5385cf034104b09b1a23ca7493d203b438736555","result":{"confidence_high":40.63829340692884,"confidence_low":29.53714518956238,"sample_count":285,"score":35.08771929824561,"score_display":"35.1","source_stated_rank":62},"run_fingerprint":"d5d8ad3b6ff24705a927294f2bbb3f7e46362a2b690bf40e35fe59a1c1b1fded","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ce154a992e1d2fb0bec85d25","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ecaa49ad622633c662d5d4d798a0a2577a5ebe8c3d26a9aea9ab184386cd8252","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b","upstream_model_label":"Qwen3.6 27B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"bcba8686eb1b301d47af1b852f210dbdb831a09d7f5f203fdcf6544309c6a39c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":74.624277,"score_display":"74.6","source_stated_rank":null},"run_fingerprint":"f4ee2b33934f53bf5ca91cbc01956e5552390420924e43f0db084c0aa7b94b9c","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_532636b8035b61ccdae69abd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eec38f3027af817b54cefab5b57c51595032842220ab8d5b4cb898dae1057efe","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b-non-reasoning","upstream_model_label":"Qwen3.6 27B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"eb95427b74cd1001c10a27442ccd7b87724522b85c88c4728218356a6eca116f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":71.734104,"score_display":"71.7","source_stated_rank":null},"run_fingerprint":"aed7739112e590a1740c35f1fb4af42844060d03537534eb591fe5a03bfe19c9","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_1dbdadbee92bb7f366d1025b","split_or_window":"standard","unit":"percent","version":"MMMU-Pro official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15","basis":"evaluation_at","evaluated_at":"2026-05-15","first_observed_at":"2026-08-29T23:52:50Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"7d26b150756e664e19b01ffded15e94319a6515d9b31908e92c8179e89fd9944","metric_details":{"num_parameters":27781427952,"pull_request":36,"result_file_url":"https://huggingface.co/Qwen/Qwen3.6-27B/resolve/refs%2Fpr%2F36/.eval_results/mmmu_pro.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/Qwen/Qwen3.6-27B"}},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9225c08e521da63df44b0ad31230f50489ffbee930c8fb4aaf2f0f8dbf6fce3f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.8,"score_display":"75.8","source_stated_rank":5},"run_fingerprint":"940ebe51d5b955c553d49f887d20fa0d489d10215de9d3f9958c200dea1036c7","source":{"content_hash":"bc590a866070c9674cba0de4af7d92af6e0921bd2db419fca54624e00d73400b","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-10-05T12:33:31Z","homepage_url":"https://mmmu-benchmark.github.io/","id":"mmmu-pro","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"MMMU-Pro","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/api/datasets/MMMU/MMMU_Pro/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_163daf9fa32fb8bd053e14eb","provider_config":{"source_id":"mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_79984934735c6e815678be85","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMMU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"72b01c3ed2f27cf7ef1d2b41f4dd03709ae27670be76c52a0fd3e4b6a1cbfc83","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMMU-Pro evaluation; prompt, task count, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"56867732d9f2c0f7b77304c985a2db6d9d0d33fe8e7e4c87aca1036339e4fe4b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.8,"score_display":"75.8","source_stated_rank":null},"run_fingerprint":"f8b3c08d346b856251d27f73c71f3513ec7b5694735df3d1e37d1c2f14ca18ac","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.6-27B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_609752b0f7bd8a6341aa27a9","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1c74287ed9e597425438bec6787bc3e8ceb651e58602e263f971b0e1eb1f96ec","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-22","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.6 27B (Non-reasoning)","source_effort":null,"source_model_version":"qwen3.6-27b_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"bf4fee08f497620015d5779ccb4ad100f29c89690690d44b245ef61d62d3761e","result":{"confidence_high":6.654528,"confidence_low":0.104738,"sample_count":71,"score":0.857142857142857,"score_display":"0.9","source_stated_rank":148},"run_fingerprint":"5d3308c2a316fa9b6e062fd5f5b59ac5dba2941cf4d7a97c855dee2eb0f93b40","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9b44263ec6b1e3d4b716a6b1ba7aea494c759ad1eb9bb43c2b9b54678c38cf79","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · critpt"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b","upstream_model_label":"Qwen3.6 27B (Reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"c104fcb6dbf3f136696892c58e5484949fa78d15caed9e959f716666105204b7","result":{"confidence_high":7.197254,"confidence_low":0.172034,"sample_count":70,"score":1.142857,"score_display":"1.1","source_stated_rank":null},"run_fingerprint":"2c58ae54af55a4d75dfb1e7a2872f56334cbc6b9fed92ab024e3401963fb9798","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_532636b8035b61ccdae69abd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c7f020f6cac0b8eeaaad98222af9d47cf0f7d0611803a4d59b45756ac31568b2","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · critpt"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b-non-reasoning","upstream_model_label":"Qwen3.6 27B (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"7e7f967c2a3d2d11a72b1bff4d6a15b9b0f0a95d5b8f96e515f961352ee25af9","result":{"confidence_high":6.724008,"confidence_low":0.10358,"sample_count":70,"score":0.857143,"score_display":"0.9","source_stated_rank":null},"run_fingerprint":"1acaa7f551017bd94cb778cfaa2ff37bb6edf9bc92b2ae7f648a1993a58f493c","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:28:28.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:28:28.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6f6daf5a5577005b49ce639be8cef9349423111c6a1b1dffce9d9730bfbbb6fe","metric_details":{"best_score_across_scorers":"0.22","model_release_date":"2026-04-22","stderr":4.163331998932266},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Wh3JxFdvxP688EWwE3t2Uk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWh3JxFdvxP688EWwE3t2Uk.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Wh3JxFdvxP688EWwE3t2Uk.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"fa104fba0f999cf030e832679191616c1df5b75b87b3a574fb9746fc5486035f","result":{"confidence_high":30.160130717907244,"confidence_low":13.839869282092758,"sample_count":100,"score":22.0,"score_display":"22.0","source_stated_rank":76},"run_fingerprint":"355216c67a71c21e9d9d96fc8ec33be93264d886ff3fc8d7c392b4beebf0780d","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e352c182e38a09e01abce653","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:28:19.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:28:19.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e2658a0803b52b62e51b15649cb6283ded38f68e6dc1672385d874ccd735faea","metric_details":{"best_score_across_scorers":"0.09","model_release_date":"2026-04-22","stderr":2.8762349126466145},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CGac6TZg5qVdxNGULzfR6R","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCGac6TZg5qVdxNGULzfR6R.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/CGac6TZg5qVdxNGULzfR6R.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e82562c5f9b02cc1b62b76588f561369f1aea8bd3fde12bc3c213064315e1a2b","result":{"confidence_high":14.637420428787365,"confidence_low":3.3625795712126356,"sample_count":100,"score":9.0,"score_display":"9.0","source_stated_rank":141},"run_fingerprint":"2db62218d0eb866b0104bbaeb481467b4871ea84270032407ffd6be8d850feff","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"116b1dd9fcdb2682e1b9e039f7b2867e25745a44a0e9e182e5370d39deaada53","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.91414,"date_is_proxy":true,"latency_seconds":1028.733,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.051},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.6-27b"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"939b9b5896abf9c2a1ab7039024566dbdd43b697b3263aeeded4973ff1dabc4e","result":{"confidence_high":74.01996,"confidence_low":65.98004,"sample_count":500,"score":70.0,"score_display":"70.0","source_stated_rank":61},"run_fingerprint":"b04c9c75024f27841e760f7696a9e9981b05d3b3d946fd630efee570b5107423","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7c98370c510777de07e3c586","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_9162beb27813d52a46a20a9f","split_or_window":"qwen-internal-agent-run","unit":"percent","version":"SWE-bench Verified · Qwen vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"d32711b53e463e7f56a1304da6f4b70b5a8c2c530683867deec03e33f105085d","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.6-27B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Verified; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"76ed278b99ffd1e5f9230954bc4ee5268b74c5c42a740c706366f11a530ecbf6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.2,"score_display":"77.2","source_stated_rank":null},"run_fingerprint":"7422f92cc687dd578dfb640a8b7a351fd402360ca813d5e49b98b53d25657372","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.6-27B column; SWE-Bench Series methodology note","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_609752b0f7bd8a6341aa27a9","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"9128225220206b5af7f3d323af5efc1bc84da52fac6411591d0294df0598b8a7","metric_details":{"model_release_date":"2026-03-22","pass_at_5":57.65765765765766,"potential_contamination":false,"sem":1.6757725439403843},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"Qwen3.6-27B__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"bb1f0b8a3d0323016acc700587394c23b4e81d4ee438cfee2e8c57d43f6756c2","result":{"confidence_high":34.45568535729433,"confidence_low":27.88665698504802,"sample_count":null,"score":31.171171171171174,"score_display":"31.17","source_stated_rank":11},"run_fingerprint":"a8ee5648c3d3e79f9d95cb5c90f39cbe20fa61ab8fa3ff19a3095c5652a3b62c","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_884552ea54f553fb70208863","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_7334bb741d8de260846d3128","split_or_window":"qwen-vendor-comparison","unit":"percent","version":"DeepSWE 1.1 · Qwen3.8 comparison run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":52,"at":"2026-08-14","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-14","status":"fresh"},"measurement_id":"225f38242c6a8b98e93a3fd3eb006df87e5ee5cc491d361e8f189cf6e9171338","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Text Performance table; Coding; DeepSWE 1.1; Qwen3.6-27B column; methodology note"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen3.8 model-card comparison on DeepSWE 1.1; temperature 1.0; top_p 0.95; 256K context; reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-14","source_content_hash_method":"raw_bytes","source_content_sha256":"57e4bdb258ee1a7d2635c5174ebd4e56abe392505cdb5f8bbb356b0dc4293641","source_published_at":"2026-08-14","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"a6e881fb08d7f02373b7ec6ca1562ac9c2dd115460a20ee27445ba6f54a80701","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.3,"score_display":"13.3","source_stated_rank":null},"run_fingerprint":"5f2dbcd294170afb51f55ec88b0f7fadbc44535eb001ded628a4aa487fba635e","source":{"content_hash":"2366d267781c2ec775c8afc831ef9ac759e918d005e3ba0e5ba4e2704a3302b8","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T16:49:13Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-27B","id":"qwen3-8-27b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Text Performance table; Coding; DeepSWE 1.1; Qwen3.6-27B column; methodology note","name":"Qwen3.8-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a26de80da0b74a12cbb6375e","provider_config":{"source_id":"qwen3-8-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_d4a5abde3ed9874198f1dc1f","split_or_window":"qwen-refined-vendor-run","unit":"percent","version":"SWE-bench Pro · Qwen refined vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"dd8f0024be2d8e7e8cbc3370fa4f9f6c3f95e2cd9b8fd2b77aa12a2e29ecd486","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.6-27B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Pro with Qwen's documented corrections/refinements to problematic tasks; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"6026abb572c5a53a08110c13a15c48d68315673ae81f11df6bff8d0515dc915b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.5,"score_display":"53.5","source_stated_rank":null},"run_fingerprint":"2fe23ed0384343355fc2a2e00dc8b270641c97033bb9d88b14e189bf4386f491","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.6-27B column; SWE-Bench Series methodology note","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_609752b0f7bd8a6341aa27a9","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_b530ca4da413885c54f52baa","split_or_window":"qwen-vendor-reported","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:24Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"64c948e386bdb00ddb89d83bd46ac3dc25dd49bc6006afc7d0be38ddaa778605","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Benchmark table; livecodebench; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"LiveCodeBench v6; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"76fab4a2249a73a0e1e012d538500edf88277ec3f4702daebc6bbf5614eec7c3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":83.9,"score_display":"83.9","source_stated_rank":null},"run_fingerprint":"c76090d044bead513df1b4f32d08f0b5ad027768854fa30d3d5a649e6ff6f80a","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Benchmark table; livecodebench; Qwen3.6-27B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_609752b0f7bd8a6341aa27a9","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"0b9fccb241fd1fa2bd1a4846b2715c31f0997fe2799705fbf1502760d5777ee3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-22","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.6 27B (Non-reasoning)","source_effort":null,"source_model_version":"qwen3.6-27b_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"8f32a451a8057bab01b362b93fb8208377ff04c6702857bd22fd3c2677f8247c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.2685185185185,"score_display":"37.3","source_stated_rank":162},"run_fingerprint":"7bc410028676280688e1f9c091a289e2a2e7a7a6381eddf2ca4a0c5976c88b9a","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ff466523813c9dcbf15808837259ef515e44e438023b522cc7ce1ebda2d4d4ef","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · scicode"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b","upstream_model_label":"Qwen3.6 27B (Reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"cce69733a0d1a1c8ac9ca2afaccfe516796b9b988b8447da7b1d98dc2cfb4e41","result":{"confidence_high":48.596504,"confidence_low":37.240562,"sample_count":288,"score":42.824074,"score_display":"42.8","source_stated_rank":null},"run_fingerprint":"717c0df15c864376e4d8b8587e12b5750ac15b4ee626f1823d62cc470104fe9d","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_532636b8035b61ccdae69abd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"11ae91b3fac562f910f02383073fa070524caeb6ee01d89e0bc88f1dddcf5430","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":4.655127,"date_is_proxy":true,"latency_seconds":9762.856,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.414},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"alibaba/qwen3.6-27b"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"e95f4271a9256ac262cfd181a9da79d6d428c1354f99ae7942100b3c8778ab30","result":{"confidence_high":18.632440000000003,"confidence_low":5.249560000000001,"sample_count":null,"score":11.941,"score_display":"11.9","source_stated_rank":88},"run_fingerprint":"a252904f3440e4e8507d58c4b57359657401e28a33ddaa23e59635fd794feff4","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0292f7dcc0ed61ad2140555d","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"mean_reward","name":"Terminal-Bench 2.0","release_id":"release_445dddec34d8f59576432f47","split_or_window":"qwen-vendor-reported","unit":"percent","version":"2.0"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:24Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"e073a48004588cbe5513f20a0d8131b276e4cfd0ccdd9b52c1458aa9b60a1a2a","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Benchmark table; terminal-bench-2; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Terminal-Bench 2.0; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"terminal agent toolset"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"37ea619732120eade3ba338d8f65be3dc4a3a4c7eb8f1bc647f13bef17081839","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":59.3,"score_display":"59.3","source_stated_rank":null},"run_fingerprint":"36ec84965d9756d7ef03aa16f5ba761af802955cbd43ccea2426ec4b0e666926","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Benchmark table; terminal-bench-2; Qwen3.6-27B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_609752b0f7bd8a6341aa27a9","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2779b590be53826c52fe96d480a70029efb191424c3fccad9f92b227012b13fa","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b","upstream_model_label":"Qwen3.6 27B (Reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"662a44299f94415dfc1e341230d74ca6dcabbd11203965171268395946a14942","result":{"confidence_high":70.178615,"confidence_low":50.286348,"sample_count":89,"score":60.674157,"score_display":"60.7","source_stated_rank":null},"run_fingerprint":"945a0cdd4f86b0ec7e88d0bec3eba2736717861def6b10f9ead39f1f3239d776","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_532636b8035b61ccdae69abd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"dec18122070a8adebac368941c904921a4ec61100cec371dc66b1a49e236205e","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b-non-reasoning","upstream_model_label":"Qwen3.6 27B (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"2f1ab963cc9f7159d2aa4b5b4bee70fde6c6a10cc6bbcc44d94a853259a79fd0","result":{"confidence_high":61.424061,"confidence_low":41.08918,"sample_count":89,"score":51.310861,"score_display":"51.3","source_stated_rank":null},"run_fingerprint":"4b87f82c1dac313d194c8733289b6beff1367e7a3b87abdeaf645c73b6eb607a","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4f87e3d5f445fc73df857e0b951cce2b627e0117e2e5d5c61982935879c0d745","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · tauBanking"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b","upstream_model_label":"Qwen3.6 27B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"498e8600e17e069a73abb7d078bcd64e4ff805ba73614f86234dd512307956bf","result":{"confidence_high":25.359201,"confidence_low":10.579935,"sample_count":97,"score":16.701031,"score_display":"16.7","source_stated_rank":null},"run_fingerprint":"b843dd6ef33f576977d995a9983c8c40a6470c4c3afaac0a90f6a3358ccefcfc","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_532636b8035b61ccdae69abd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"71fb929390e6dcf2cf37ebea0447077782305ac029061bbf792c505529ac1a87","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · tauBanking"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-27b-non-reasoning","upstream_model_label":"Qwen3.6 27B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"9506d96a6f8354ca583dd6c76627dfc07b6ff1a6d1f439f86448d268691d009d","result":{"confidence_high":16.70105,"confidence_low":4.958266,"sample_count":97,"score":9.278351,"score_display":"9.3","source_stated_rank":null},"run_fingerprint":"e64e672ab51c18df04fef9fdc3a604278cb17a73730bb756bcff7e065ceaf8aa","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_5af9fe5a9d5e275323798c8b","split_or_window":"self-contained-78-no-api-subset","unit":"percent","version":"SkillsBench · Qwen 78-task self-contained subset"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"596c96a04adbcb7e836644982f0cf40987a22d7b317f5814b1a7c369d1847786","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 78 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.6-27B column; SkillsBench methodology note"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen vendor evaluation through OpenCode on 78 self-contained tasks, excluding API-dependent tasks; average of five runs; exact tool policy and reasoning mode not disclosed","question_count":78,"reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":5,"scaffold":"OpenCode","tools_allowed":"not reported"},"protocol_fingerprint":"0075a32b62b602932d1572b58a2560fe97ac92d58166e3cd33e67ab077a6bac5","result":{"confidence_high":59.110556,"confidence_low":37.458426,"sample_count":78,"score":48.2,"score_display":"48.2","source_stated_rank":null},"run_fingerprint":"9aded900bc59f172374df6f9adce1e88e1babc87c4cd2e4cce3ca83eec8a1290","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.6-27B column; SkillsBench methodology note","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_609752b0f7bd8a6341aa27a9","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifeval","metric":"accuracy","name":"IFEval","release_id":"release_782921d592a5244ea0b5aaf4","split_or_window":"demietrich-ifeval-subset-150","unit":"percent","version":"IFEval"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":87,"at":"2026-07-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-10","status":"fresh"},"measurement_id":"99900f75d025eaf94da951f732abd09ca1f224929527d4af677e0827ac4a719c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 150 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://demietrich.com/blog/qwen36-quant-shootout/","evidence_sha256":"daf77d9e65cabe65e8ce1fc168d110a690afaef00ba07145a60bfd5c23833445","kind":"reviewed-static-report","published_at":"2026-07-10","reviewed_at":"2026-09-05","source_url":"https://demietrich.com/blog/qwen36-quant-shootout/","version_id":"2716ecb401c028b19783edcd97a485ca56b471a3f34a014a46ccea63562eff89"},"source_locator":"Benchmark table; ifeval; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["ifeval_protocol_not_uniform","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Demietrich","evidence_verified":true,"freshness_proxy":true,"protocol":"IFEval; 150-sample subset; greedy temperature 0, thinking disabled as reported; llama.cpp; 2x RTX3090 host. Exact subset IDs and checkpoint revision not published.","quantization":{"backend":"llama.cpp","base_checkpoint":"Qwen/Qwen3.6-27B","hardware":"2 × RTX 3090 host; 1 GPU for Q4","id":"q4-k-xl","label":"Q4_K_XL","limitations":"Small subsets; backend and GPU count vary. Compare complete configurations, not quantization alone. Exact checkpoint revisions and per-question answers are undisclosed.","reference":false,"study_id":"demietrich-qwen36-quant","study_label":"Demietrich · 2026-07-10"},"question_count":150,"reasoning_effort":"off","result_published_at":"2026-07-10","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"78c3c033f6360ceef5650d3f5954e180b18c56492325613e3248e7cf69cb8c48","source_published_at":"2026-07-10","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d722c1d82ccd8a43a15bfae04839e6637714fbe32affdb18dbaad6e60da7d4bc","result":{"confidence_high":90.000824,"confidence_low":78.641208,"sample_count":150,"score":85.2,"score_display":"85.2","source_stated_rank":null},"run_fingerprint":"470445e7254ced6d0be5c6878604526309d89dc7d114987655632bbe7b4c35d2","source":{"content_hash":"e3e3e61d0f0166b17263ff0c90a3945dee11a937b8942d7c7dc4047d272420bd","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://demietrich.com/blog/qwen36-quant-shootout/","id":"demietrich-qwen36-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; ifeval; Qwen3.6-27B column","name":"Demietrich Qwen3.6-27B quantization study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://demietrich.com/blog/qwen36-quant-shootout/"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifeval","metric":"accuracy","name":"IFEval","release_id":"release_782921d592a5244ea0b5aaf4","split_or_window":"demietrich-ifeval-subset-150","unit":"percent","version":"IFEval"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":87,"at":"2026-07-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-10","status":"fresh"},"measurement_id":"1983f490d236fa6b7d1cf8ee0c54595d9871449a390e08a9a7ff612697e7cbde","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 150 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://demietrich.com/blog/qwen36-quant-shootout/","evidence_sha256":"daf77d9e65cabe65e8ce1fc168d110a690afaef00ba07145a60bfd5c23833445","kind":"reviewed-static-report","published_at":"2026-07-10","reviewed_at":"2026-09-05","source_url":"https://demietrich.com/blog/qwen36-quant-shootout/","version_id":"2716ecb401c028b19783edcd97a485ca56b471a3f34a014a46ccea63562eff89"},"source_locator":"Benchmark table; ifeval; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["ifeval_protocol_not_uniform","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Demietrich","evidence_verified":true,"freshness_proxy":true,"protocol":"IFEval; 150-sample subset; greedy temperature 0, thinking disabled as reported; llama.cpp; 2x RTX3090 host. Exact subset IDs and checkpoint revision not published.","quantization":{"backend":"llama.cpp","base_checkpoint":"Qwen/Qwen3.6-27B","hardware":"2 × RTX 3090 host; 1 GPU for Q4","id":"q6-k-xl","label":"Q6_K_XL","limitations":"Small subsets; backend and GPU count vary. Compare complete configurations, not quantization alone. Exact checkpoint revisions and per-question answers are undisclosed.","reference":true,"study_id":"demietrich-qwen36-quant","study_label":"Demietrich · 2026-07-10"},"question_count":150,"reasoning_effort":"off","result_published_at":"2026-07-10","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"78c3c033f6360ceef5650d3f5954e180b18c56492325613e3248e7cf69cb8c48","source_published_at":"2026-07-10","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8b3a2f5dad46da839b009b97670885db0728f650261f5afa320a540642db3421","result":{"confidence_high":88.420692,"confidence_low":76.51623,"sample_count":150,"score":83.3,"score_display":"83.3","source_stated_rank":null},"run_fingerprint":"4825afe47680d54ae9928c1c78f56c2b5ecc2632b0037415765185d8c42d958b","source":{"content_hash":"e3e3e61d0f0166b17263ff0c90a3945dee11a937b8942d7c7dc4047d272420bd","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://demietrich.com/blog/qwen36-quant-shootout/","id":"demietrich-qwen36-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; ifeval; Qwen3.6-27B column","name":"Demietrich Qwen3.6-27B quantization study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://demietrich.com/blog/qwen36-quant-shootout/"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifeval","metric":"accuracy","name":"IFEval","release_id":"release_782921d592a5244ea0b5aaf4","split_or_window":"demietrich-ifeval-subset-150","unit":"percent","version":"IFEval"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":87,"at":"2026-07-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-10","status":"fresh"},"measurement_id":"b0594a377c43c367f4e9bc852679f7670db5b0aa30dcedb1d12339578322c2da","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 150 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://demietrich.com/blog/qwen36-quant-shootout/","evidence_sha256":"daf77d9e65cabe65e8ce1fc168d110a690afaef00ba07145a60bfd5c23833445","kind":"reviewed-static-report","published_at":"2026-07-10","reviewed_at":"2026-09-05","source_url":"https://demietrich.com/blog/qwen36-quant-shootout/","version_id":"2716ecb401c028b19783edcd97a485ca56b471a3f34a014a46ccea63562eff89"},"source_locator":"Benchmark table; ifeval; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["ifeval_protocol_not_uniform","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Demietrich","evidence_verified":true,"freshness_proxy":true,"protocol":"IFEval; 150-sample subset; greedy temperature 0, thinking disabled as reported; vLLM; 2x RTX3090 host. Exact subset IDs and checkpoint revision not published.","quantization":{"backend":"vLLM","base_checkpoint":"Qwen/Qwen3.6-27B","hardware":"2 × RTX 3090 host; 1 GPU for Q4","id":"fp8","label":"FP8","limitations":"Small subsets; backend and GPU count vary. Compare complete configurations, not quantization alone. Exact checkpoint revisions and per-question answers are undisclosed.","reference":false,"study_id":"demietrich-qwen36-quant","study_label":"Demietrich · 2026-07-10"},"question_count":150,"reasoning_effort":"off","result_published_at":"2026-07-10","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"78c3c033f6360ceef5650d3f5954e180b18c56492325613e3248e7cf69cb8c48","source_published_at":"2026-07-10","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"83a4b840e510909a1244b119600e723ee2e10eb3ad1d2efab133577e473b08d3","result":{"confidence_high":88.420692,"confidence_low":76.51623,"sample_count":150,"score":83.3,"score_display":"83.3","source_stated_rank":null},"run_fingerprint":"38173ea557efaa9b88378ee83f596f5670655f52bcfb41c48825caaac99e6c02","source":{"content_hash":"e3e3e61d0f0166b17263ff0c90a3945dee11a937b8942d7c7dc4047d272420bd","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://demietrich.com/blog/qwen36-quant-shootout/","id":"demietrich-qwen36-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; ifeval; Qwen3.6-27B column","name":"Demietrich Qwen3.6-27B quantization study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://demietrich.com/blog/qwen36-quant-shootout/"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifeval","metric":"accuracy","name":"IFEval","release_id":"release_782921d592a5244ea0b5aaf4","split_or_window":"demietrich-ifeval-subset-150","unit":"percent","version":"IFEval"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":87,"at":"2026-07-10","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-10","status":"fresh"},"measurement_id":"5615313839e567f56debee2cd20d65b8fcdcca75e6c2bf1d6bb2ac9e3d31618f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 150 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://demietrich.com/blog/qwen36-quant-shootout/","evidence_sha256":"daf77d9e65cabe65e8ce1fc168d110a690afaef00ba07145a60bfd5c23833445","kind":"reviewed-static-report","published_at":"2026-07-10","reviewed_at":"2026-09-05","source_url":"https://demietrich.com/blog/qwen36-quant-shootout/","version_id":"2716ecb401c028b19783edcd97a485ca56b471a3f34a014a46ccea63562eff89"},"source_locator":"Benchmark table; ifeval; Qwen3.6-27B column"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["ifeval_protocol_not_uniform","quantization_study_not_global_comparable"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-quantization-study","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Demietrich","evidence_verified":true,"freshness_proxy":true,"protocol":"IFEval; 150-sample subset; greedy temperature 0, thinking disabled as reported; vLLM; 2x RTX3090 host. Exact subset IDs and checkpoint revision not published.","quantization":{"backend":"vLLM","base_checkpoint":"Qwen/Qwen3.6-27B","hardware":"2 × RTX 3090 host; 1 GPU for Q4","id":"int4-autoround","label":"INT4-AutoRound","limitations":"Small subsets; backend and GPU count vary. Compare complete configurations, not quantization alone. Exact checkpoint revisions and per-question answers are undisclosed.","reference":false,"study_id":"demietrich-qwen36-quant","study_label":"Demietrich · 2026-07-10"},"question_count":150,"reasoning_effort":"off","result_published_at":"2026-07-10","source_content_hash_method":"html_visible_body_text_v2","source_content_sha256":"78c3c033f6360ceef5650d3f5954e180b18c56492325613e3248e7cf69cb8c48","source_published_at":"2026-07-10","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"85d852890ebf4d711bb73a97400ead2b45690207f38f71cb072c3b131a31b626","result":{"confidence_high":86.216737,"confidence_low":73.650035,"sample_count":150,"score":80.7,"score_display":"80.7","source_stated_rank":null},"run_fingerprint":"d2562c34d693d747df31272b6383ffeb80647e06c48cf28570cf2af52a37a546","source":{"content_hash":"e3e3e61d0f0166b17263ff0c90a3945dee11a937b8942d7c7dc4047d272420bd","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T16:42:26Z","homepage_url":"https://demietrich.com/blog/qwen36-quant-shootout/","id":"demietrich-qwen36-quant","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; ifeval; Qwen3.6-27B column","name":"Demietrich Qwen3.6-27B quantization study","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://demietrich.com/blog/qwen36-quant-shootout/"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-quantization-study","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":3,"source_authority_tier":"C","trust_rank":3,"trust_tier":"C","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d4b12acc84c761b8bfdb1f4d7b53534bd1a8dccf3e687cff76a4f83e844912ba","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-22","reported_confidence_interval":"-22 / +22","source_locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-6-27b","upstream_model_label":"Qwen3.6 27B (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"2bba1f8868154da6547b0a3f5ffc459ef13a5fc243fde03cfb17d6e3aa0f5ca0","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":973.34,"score_display":"973","source_stated_rank":null},"run_fingerprint":"35f5016207f05d35161999178bb6fa57915b3da34747b016e4fa9ad1a8c9c921","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_532636b8035b61ccdae69abd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"96fb0249030b5c9bd7f2adfc401b563ac74a0362e3171f19a52ea8da423fb8c9","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-22","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-6-27b","name":"Qwen 3.6 27B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-6-27b-non-reasoning","upstream_model_label":"Qwen3.6 27B (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"39f5436208274f14cf740dbb6ce9613e3a6ab2bff6c62b62e6a3f5fb8f80440f","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":949.9,"score_display":"950","source_stated_rank":null},"run_fingerprint":"8a93b3cadb1203393c845b4852d9e7502dc728b87eaeef470956813517a4610d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 27B (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_2cd29242487b8e61b4d8196f","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6c516f98e962977829564a0fb1a41eb6d8de396392f93a842226748a251bff65","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-14"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen 3.6 35B-A3B; model release: 2026-04-14","source_accessibility":"Open weights (unrestricted)","source_model_name":"Qwen 3.6 35B-A3B","source_model_release_date":"2026-04-14","source_model_versions":["qwen3.6-35b-a3b","qwen3.6-35b-a3b_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"b341d1e53567ed9e35b7ac8908d0259dfdbe08e7398f079edf10d57af753b98b","result":{"confidence_high":145.97,"confidence_low":141.2,"sample_count":null,"score":143.92,"score_display":"143.92","source_stated_rank":90},"run_fingerprint":"4a7fb1aea9224a634b1a5314d7f81f50b7fe8b69816abad9e576eb2d418f5276","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ec15963478c8f005890aa25e","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"753bdb91024211846b77712553068b75f7df5b64e8fc6a8f996aed453897e6f7","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b","upstream_model_label":"Qwen3.6 35B A3B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"d7c42e862df4b14789f99d232d6c205024355686ef13d96cdfa5ac01401c0fea","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.22899,"score_display":"18.2","source_stated_rank":null},"run_fingerprint":"39127bd1230d18229b5162d611b2da7d06d21674f8072f443d937a657d1d0b79","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_fa62330ab4b31843f371c58b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"59e296c22ab6fcce81b6d711ab9153dd0d7f3ae4b609f5bc5261955d7a7c1957","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.6 35B A3B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"beed0e44781b1bd5ed5739a29a75f12ed2bacf53991fa3284e1be310a9cd0e0e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":15.232077,"score_display":"15.2","source_stated_rank":null},"run_fingerprint":"95dc4c3bc4b93b3cf5ff563c54b58ba848c704c6711d4711f000406fc91655a9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"78e8b6b4afc687fe6ea72cced01ce3098cbca2e32e30a0821c0e1f8e2d589b44","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":null,"num_parameters":35951822704,"pull_request":3,"result_file_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B/resolve/refs%2Fpr%2F3/.eval_results/hle.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"}},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":21.4,"score_display":"21.4","source_stated_rank":64},"run_fingerprint":"295bb884c1d4b0d768c3da8c30d4beb4629af490be68b4c44daa000a802e1436","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e4a5a6dee408943601c0a8aa","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_287aa4bcb31b5b1ba26e562a","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"cf008702d9df18a9f52a08721f64c7a8d68e78ad6b714b3cce354c1246ee58fe","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.6-35B-A3B column"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE evaluation; task count, split, tools, judge, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"4cacf5cf831c2e7165c8e718c71edb9fcd7e427e4ed4aeb02385a7200d28ca81","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":21.4,"score_display":"21.4","source_stated_rank":null},"run_fingerprint":"7dc4953ce407600a6f904a0d605b893fbc41fd774fe8e19d64223380aff5481c","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.6-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_771e537906e49bf2223bf52f","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3ee8e083a2eeaa17da445c1ce18e915e7c35ce0a8214be973b8565b76cbde3a2","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · hle"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b","upstream_model_label":"Qwen3.6 35B A3B (Reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2dcd95e55f5307124f729c813f931e7c4156aa5399c423dc4817906973dad29e","result":{"confidence_high":24.045946,"confidence_low":20.538337,"sample_count":2158,"score":22.242817,"score_display":"22.2","source_stated_rank":null},"run_fingerprint":"0d471cfc57bfbe237fced3bdf0aa9b91f08a58377335c2d4a3d90a3c47657059","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_fa62330ab4b31843f371c58b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a6b9d0e7650730f4356a2ffc2c85a46bf5941c30145156073dfdef02cd98dcbc","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · hle"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.6 35B A3B (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"e8c2927b07b4977f94b1fee230255a4139670bbb9251b42fd3b191db29e61cbb","result":{"confidence_high":15.425716,"confidence_low":12.506099,"sample_count":2158,"score":13.901761,"score_display":"13.9","source_stated_rank":null},"run_fingerprint":"b989078112f508156c57f43bcdaf1ddddbb66f7664a42a1fe44ff5662d806911","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"89ecf04ed44bc0f2d153db0ad78b425db7aa389911b60109f1cf530da6cd0738","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b","upstream_model_label":"Qwen3.6 35B A3B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"cd86902db962252dba3e202bda422ce0bce09929b0380067bf349a8b527063d4","result":{"confidence_high":88.572835,"confidence_low":78.410383,"sample_count":198,"score":84.141414,"score_display":"84.1","source_stated_rank":null},"run_fingerprint":"abc070321e4f3f281dab18a95bf7ea8663c787db94229e8142ff8408cc5eaf9c","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_fa62330ab4b31843f371c58b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4d303a9f5107fdf3040e726ede5226c2d08327f90ac68980cb2d49d2086c1c46","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.6 35B A3B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"927fd01b03fd7c634c322a8334f1f4d2969f68b4281576387c0fa1a114971e91","result":{"confidence_high":86.480045,"confidence_low":75.746969,"sample_count":198,"score":81.717172,"score_display":"81.7","source_stated_rank":null},"run_fingerprint":"80ddc8d0dbbc6cbb2518b1502d66536d454ccc099cd42e9ae60d60342717afd3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:12:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:12:45.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e843ada8aff80d80861e470177daf7847a05b3f8cf7b6e38b9f6655576dc41a5","metric_details":{"best_score_across_scorers":"0.8383838383838383","model_release_date":"2026-04-14","stderr":2.622591986362927},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"EWAyxFsRUvuu4NpmmLqWkn","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEWAyxFsRUvuu4NpmmLqWkn.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/EWAyxFsRUvuu4NpmmLqWkn.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"74a6c8e47e83fa6b57031fb378bc94e8a955916a64d9c02cc4b54b8ed38c42b3","result":{"confidence_high":88.97866413165517,"confidence_low":78.6981035451125,"sample_count":null,"score":83.83838383838383,"score_display":"83.8","source_stated_rank":99},"run_fingerprint":"177fe13125d51fbdc4756cf48391810596fe984a4e778e14e1b0b874db9ffe04","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e7dfb666e9374aca0ff98037","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:12:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:12:39.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e4e482464077810290fe2f1f999fd8a116458d6b8cb9665fcf2ca1e4d4c7a998","metric_details":{"best_score_across_scorers":"0.8484848484848485","model_release_date":"2026-04-14","stderr":2.554565042660364},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"9kErpmgQ42wTvgJaFn478v","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9kErpmgQ42wTvgJaFn478v.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/9kErpmgQ42wTvgJaFn478v.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"b48fb624cf688d4a2fb504472fb457ff2153e1cf0121610bf1fbb63184130f9a","result":{"confidence_high":89.85543233209916,"confidence_low":79.84153736487053,"sample_count":null,"score":84.84848484848484,"score_display":"84.8","source_stated_rank":92},"run_fingerprint":"c413cdc0c513cd7d0de013f7e3a2b2e2fbfa419fc793a317202b9622f4ac59ce","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_95dc04609d027072fa229d3b","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMLU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"56025d5f422d3e5038eef6ead2713c289c780a06680a3d3945ecc5b5b549fd3b","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.6-35B-A3B column"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMLU-Pro evaluation; prompt, shots, task count, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"70c7bbff02e177f65267ba3c33be7b6f38096fbaac03c9cc970c7210519303fa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.2,"score_display":"85.2","source_stated_rank":null},"run_fingerprint":"b3f100c62868a333f56d5066df5805ed52221f39edcd19b44a6eab2ef379e226","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.6-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_771e537906e49bf2223bf52f","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:03:16.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:03:16.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3c9f0ccf870547cffd59b94fc3dc84d4258db834a808ca482d08a130ddb8f74c","metric_details":{"best_score_across_scorers":"0.8666666666666667","model_release_date":"2026-04-14","stderr":5.124707431905383},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"3n5KDcBaKqJh3WTg7qU2rs","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3n5KDcBaKqJh3WTg7qU2rs.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/3n5KDcBaKqJh3WTg7qU2rs.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8b33069dabc4dc7d46ccd6d6ba796cc168c2e534203b12e04d74fff3b0ea4446","result":{"confidence_high":96.71109323320123,"confidence_low":76.62224010013212,"sample_count":45,"score":86.66666666666667,"score_display":"86.7","source_stated_rank":82},"run_fingerprint":"1792ca054cdce6e0d6bd1da8b982aa5cdfe63c8e2d53b1dd40321036cf33f84e","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_20fa5f508edb624d3a1dc2c8","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:03:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:03:06.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a1341b2ae3fa9eea0a706b71fe76271b6238ef3b166c5b9f41ea8d32c5174639","metric_details":{"best_score_across_scorers":"0.6888888888888889","model_release_date":"2026-04-14","stderr":6.979205927323111},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7GiYWM7UagUsHpNsdymd4z","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7GiYWM7UagUsHpNsdymd4z.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/7GiYWM7UagUsHpNsdymd4z.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"2e81ae8686d2f59f8a77231393215853fbfd314691ac903b137a06b750b0291d","result":{"confidence_high":82.56813250644218,"confidence_low":55.20964527133559,"sample_count":45,"score":68.88888888888889,"score_display":"68.9","source_stated_rank":140},"run_fingerprint":"fe0ce1fe67c1801e8c4ea3cf894c27a0e52478d17ce0aba6f69f845f9886d639","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:34:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:34:37.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"89dd4913f8dc1978d6eab60f3216f34bf2893de7753355195105b98ee97c300f","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.20350877192982456","leaderboard_private_problem_count":285,"model_release_date":"2026-04-14","public_example_count":10,"stderr":2.389037106974561},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"YRvn36dmMyDbS5e4XkF4Xi","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYRvn36dmMyDbS5e4XkF4Xi.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YRvn36dmMyDbS5e4XkF4Xi.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.6 35B-A3B","thinking":null,"upstream_model_id":"qwen3.6-35b-a3b_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"b4690aeb54cab157351a24640565f368ae0d16229c6fab0cf18e044e124a89f5","result":{"confidence_high":25.033389922652596,"confidence_low":15.668364463312315,"sample_count":285,"score":20.350877192982455,"score_display":"20.4","source_stated_rank":83},"run_fingerprint":"afcaaf7566673fde1b59015c688f79fe52961fd82a75806c8b3f217eaa9f0357","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-29T11:06:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-29T11:06:39.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b34247daf40f3751e549a0b87fbcfa036b8f67421611dafd39d95eab4702a871","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.17543859649122806","leaderboard_private_problem_count":285,"model_release_date":"2026-04-14","public_example_count":10,"stderr":2.2569134425265194},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"ggG3n6WVPZn7Gu2RNyfQbK","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FggG3n6WVPZn7Gu2RNyfQbK.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ggG3n6WVPZn7Gu2RNyfQbK.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.6 35B-A3B","thinking":null,"upstream_model_id":"qwen3.6-35b-a3b","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"40f7eae39b79bcc852c6bdad12b5061897d12fd476aae5cdd35a76af9eb2c682","result":{"confidence_high":21.96740999647478,"confidence_low":13.120309301770828,"sample_count":285,"score":17.543859649122805,"score_display":"17.5","source_stated_rank":94},"run_fingerprint":"8fde9f55cb513f06e77ed764ada7ba5bda5afd7ce6c2b535c1c02cb907cada55","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1cb0db7f5f8f53d5b87c754c","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c4dc1c32c4a6430b02b8f6f0e26024b36e99035122086cd780c3078dc64c190a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b","upstream_model_label":"Qwen3.6 35B A3B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"db930b98834e9fe06a6a12ef374a9cdc2afcc34543352434b4d9fd91caa27d2d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.028902,"score_display":"75.0","source_stated_rank":null},"run_fingerprint":"aa261f074d98b05020ed0e8f2b10f8e2379b12474910f5a615d18908c9249125","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_fa62330ab4b31843f371c58b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"32854f5a4eeb0302f60dff39c7c3e387821c7be2a5aeb89326c870a03d84334c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.6 35B A3B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2b9498e525db2425238a8c7dc16b935049b3b8c9b8da2c4fcf85399db8102b9e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":71.040462,"score_display":"71.0","source_stated_rank":null},"run_fingerprint":"ce0a1bd0f1abec33548d87ce6bd45e70f742047ebd230194ad797ba28f9c93eb","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_1dbdadbee92bb7f366d1025b","split_or_window":"standard","unit":"percent","version":"MMMU-Pro official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15","basis":"evaluation_at","evaluated_at":"2026-05-15","first_observed_at":"2026-08-29T23:52:50Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"95a84b499e9d4f1e893a817e75410f51b4d75533e5a087327e3ea2b960c5ad25","metric_details":{"num_parameters":35951822704,"pull_request":59,"result_file_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B/resolve/refs%2Fpr%2F59/.eval_results/mmmu_pro.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"}},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9225c08e521da63df44b0ad31230f50489ffbee930c8fb4aaf2f0f8dbf6fce3f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.3,"score_display":"75.3","source_stated_rank":6},"run_fingerprint":"76cda1382c54a7741f5ec034a0d73b5e1e4aa9444de212ca012e4fd4c1d15e5f","source":{"content_hash":"bc590a866070c9674cba0de4af7d92af6e0921bd2db419fca54624e00d73400b","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-10-05T12:33:31Z","homepage_url":"https://mmmu-benchmark.github.io/","id":"mmmu-pro","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"MMMU-Pro","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/api/datasets/MMMU/MMMU_Pro/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5822021443d268c64baddbeb","provider_config":{"source_id":"mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_79984934735c6e815678be85","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMMU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"1eb010c4c4398997417414966fe8b23c50134864094b4ccebe4554df9d47fbfc","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.6-35B-A3B column"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMMU-Pro evaluation; prompt, task count, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"88ff021a3a75ecf4d3dcf90fa813de6de1b99cc0fac3e1a9502d28e8f43a47f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":75.3,"score_display":"75.3","source_stated_rank":null},"run_fingerprint":"41242c23e4bb6a3622c1007812a72aa4f57ff0fef7b18088908c4ddf8402bcff","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.6-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_771e537906e49bf2223bf52f","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b8176b571122899a6bde093fb0e7f1c1c75fec1b28a58db8fdf8ff4451ed1db8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-14","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.6 35B A3B (Non-reasoning)","source_effort":null,"source_model_version":"qwen3.6-35b-a3b_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"6de27bf2f34049fa14234cab6a3d74f4017d0be8a98cfeb452d9261292b3e7b9","result":{"confidence_high":5.132974,"confidence_low":0.0,"sample_count":71,"score":0.0,"score_display":"0.0","source_stated_rank":167},"run_fingerprint":"f37ab364627ac85ffe61682160b14edc17aa7778527fddf267e22e06b6227135","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9f7c51a070547a303e7304cc5c426c6e0884c3631fb8f12893ffe29c71885e09","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-14","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.6 35B A3B (Reasoning)","source_effort":null,"source_model_version":"qwen3.6-35b-a3b","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"28566d24c78040c2e671468ec8fe8b82374fef7183b6c28ace63ff8ddba29688","result":{"confidence_high":5.661392,"confidence_low":0.013679,"sample_count":71,"score":0.28571428571428603,"score_display":"0.3","source_stated_rank":156},"run_fingerprint":"aae066a603233f3f1fe9740907170948bfd1e51a331b6a36ee99fc5440a08567","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e60e3bcb6fc5df467d48657b","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0d407d3af0524eb93ef4ee54d9d35207fd29685ff776cfec57df92baf3b06f26","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · critpt"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b","upstream_model_label":"Qwen3.6 35B A3B (Reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"25de08fdb74bb5e9f1bc655ca01703a469b6572410b7c3aebf79eaa72762561a","result":{"confidence_high":5.730683,"confidence_low":0.013504,"sample_count":70,"score":0.285714,"score_display":"0.3","source_stated_rank":null},"run_fingerprint":"baf3b6d7041365b448589477037df2aa67d9695d9c70c0e8132b62d649965c90","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_fa62330ab4b31843f371c58b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0f60d7229e3bb6c6dd967d53dc12b2e4780f4d73a7c381171e3c163496ae9975","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · critpt"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.6 35B A3B (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"9515df637ed7f128639321f39687cce6762f3bc27d69dc6bd1da804df0d72b1f","result":{"confidence_high":5.202487,"confidence_low":0.0,"sample_count":70,"score":0.0,"score_display":"0.0","source_stated_rank":null},"run_fingerprint":"6850219b9f84967714547ee2aa89c8ed565ec2cb77d142e2c481b990b1297f6f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:16:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:16:31.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6411d78758bd11fc39326ef08a2d996482be8aafd71c35f4ecfb4c4fa754cf3a","metric_details":{"best_score_across_scorers":"0.26","model_release_date":"2026-04-14","stderr":4.4084400227680804},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"3y52KdaFo7gQZCccCQnNxp","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3y52KdaFo7gQZCccCQnNxp.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/3y52KdaFo7gQZCccCQnNxp.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7000f292bbab3d232c6a524c88da59178d1e76d76a2abe4a91fd989bf44a22d3","result":{"confidence_high":34.64054244462544,"confidence_low":17.35945755537456,"sample_count":100,"score":26.0,"score_display":"26.0","source_stated_rank":62},"run_fingerprint":"6990c385e4740c77f02f28642101bf2c825df512ae5ea6ce87db8783d8107d62","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e785c9c4450b074381f8a14b","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:16:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:16:31.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1b44d96ff51079c3d8c7e258ce9f90bf65399048ba12b2217994d0602aa8cda4","metric_details":{"best_score_across_scorers":"0.04","model_release_date":"2026-04-14","stderr":1.9694638556693238},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"g4iGoqwjeF5vqkdvtyytx5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fg4iGoqwjeF5vqkdvtyytx5.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/g4iGoqwjeF5vqkdvtyytx5.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"94051b1b1ae4e89e1bbdebdfdfbbbca3f3d0eca86fad74a83aaa71025d9a5ece","result":{"confidence_high":7.8601491571118745,"confidence_low":0.13985084288812555,"sample_count":100,"score":4.0,"score_display":"4.0","source_stated_rank":169},"run_fingerprint":"0bb88b99075ab96b6dd4b3455ed6f7678cc7023af1bf6a4536446406e794cbea","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_9162beb27813d52a46a20a9f","split_or_window":"qwen-internal-agent-run","unit":"percent","version":"SWE-bench Verified · Qwen vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"224c0cb53943005091adb7877073c5fe77eeb15b2fe1e90b5ca844041e515ec4","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.6-35B-A3B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Verified; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"dad3fccd5f7c7052561a2997a748ad0d5aa39550e8421c973d8656bf054626e7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.4,"score_display":"73.4","source_stated_rank":null},"run_fingerprint":"a4416af63b6b3d09d84ba5e68c68af087f3b48f44b4fccf015387f058218bb32","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.6-35B-A3B column; SWE-Bench Series methodology note","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_771e537906e49bf2223bf52f","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_b25eae1fad6b7076bdf70d48","split_or_window":"2026-05-15/2026-07-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":96,"at":"2026-07-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"277f0c7209140553e497ec74810fd8734d884598d12ea6206e87ff3a4c868e76","metric_details":{"model_release_date":"2026-03-14","pass_at_5":43.24324324324324,"potential_contamination":false,"sem":0.7853871970343559},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent_version":"tools","selection_rule":"current-window","upstream_model_id":"Qwen3.6-35B-A3B__tools","window_from":"2026-05-15","window_status":"current","window_to":"2026-07-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"a399d27e825618cb353e8e50bf75dcd706952ca7575dce1096ed9443bf37b7e7","result":{"confidence_high":26.22404359087202,"confidence_low":23.145325778497345,"sample_count":null,"score":24.684684684684683,"score_display":"24.68","source_stated_rank":12},"run_fingerprint":"bcc8e41dde7f4b66d8ec829c320dd9761819a8a5c98ef9d10a7318c33383acf4","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f377cfdc5fe45182122cd035","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_d4a5abde3ed9874198f1dc1f","split_or_window":"qwen-refined-vendor-run","unit":"percent","version":"SWE-bench Pro · Qwen refined vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"184e91d988f81a6243e86613f131a71debf580278e2ea43760c22e34d9315448","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.6-35B-A3B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Pro with Qwen's documented corrections/refinements to problematic tasks; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"5ea251f2329b35827abdf02131617fdfb607ea4f7d8ed5e602b3e023e39868d3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":49.5,"score_display":"49.5","source_stated_rank":null},"run_fingerprint":"54b9895511035cf904ba6072602a820a6f9b4b8f8a018283f4667b14880f6a6f","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.6-35B-A3B column; SWE-Bench Series methodology note","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_771e537906e49bf2223bf52f","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_b530ca4da413885c54f52baa","split_or_window":"qwen-vendor-reported","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:24Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"155dae3ff9ef5db154bec3fe3edff55835c072f94dbe16df72e696ffc9ff9ad4","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Benchmark table; livecodebench; Qwen3.6-35B-A3B column"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"LiveCodeBench v6; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c5897a9b1049c59b6cd6b78df57d0085d6d80e4198b3402492e385b10f29623f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.4,"score_display":"80.4","source_stated_rank":null},"run_fingerprint":"a3203eca6c4217aa2021c6b546a8d783f3d9fb86d5e0fa8725630b7ccfaf72c8","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Benchmark table; livecodebench; Qwen3.6-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_771e537906e49bf2223bf52f","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fbf2e4dda2839c772a00916866522494502f5031697d276a4f33f897c6f4da08","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-14","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.6 35B A3B (Non-reasoning)","source_effort":null,"source_model_version":"qwen3.6-35b-a3b_none","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"632d7382d677ba9441d1c041ccbcc1ad0ecef8bb10d0ffc1418b96df86047ce0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1.27314814814815,"score_display":"1.3","source_stated_rank":201},"run_fingerprint":"69f537bd1bf05340023481ad1567606df20d10b9cab56ee340246af68db67dee","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1c71951a8274be531b9ad10a24b7415d8ff1b7de4d20c686a3c48880ea65d4d6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-14","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.6 35B A3B (Reasoning)","source_effort":null,"source_model_version":"qwen3.6-35b-a3b","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"766e63482093f225aa48b8c9dcd6ae1401800e990b7a82beb625e35e35b8bbf7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.7638888888889,"score_display":"35.8","source_stated_rank":168},"run_fingerprint":"dc2a33de0fea318b9273f4e18ab2b8353b8d47c1bc5e1101ab1a09c90b25b33a","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9f8b471c8142f59bb8b69a15","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"67424e0d1476404b2de4c0d3d0d126a55edd0ad397356183feacd4073b267f19","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · scicode"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b","upstream_model_label":"Qwen3.6 35B A3B (Reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"5b67a2aef917b1835328a24f0dbc408477274065b9b17aed07c9a8b4e051602e","result":{"confidence_high":42.279523,"confidence_low":31.222084,"sample_count":288,"score":36.574074,"score_display":"36.6","source_stated_rank":null},"run_fingerprint":"e6f1c16f090dd2ee8d5c1ca8148fccccc92e70f90b18b081f08eee79ec1ea179","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_fa62330ab4b31843f371c58b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5e3e0aed62f3b19b70bfd48631dca67ab637dea09072978e56514604529b63e5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-14","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.6-35b-a3b","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"c9662d1a8fa00d0b928c4439fae85fbe4d0b0b5bf5f3915e743eece8b290e8f5","result":{"confidence_high":1268.05,"confidence_low":1232.25,"sample_count":1535,"score":1250.15,"score_display":"1250.15","source_stated_rank":121},"run_fingerprint":"395f11c8c08801dd0a9fd9d9715cb5127e335e3e48aeef3dde8aa7994e6fea24","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7831c3668b6ad545a4b181f8","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ce5b46db92f3eb70853cf8cdc50daf8e7e5345bdd350127fd696eac79344c787","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-14","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"qwen3.6-35b-a3b","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"c6c795b7e9a1dbcd297aa30fcc438a0b5d01f63aeb58081a4b8c2776603b447f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.49,"score_display":"34.5","source_stated_rank":143},"run_fingerprint":"180b8f5eaef20edaf9fdd56dcb312de7daf390aea2b9eaf3794ef58c548b435b","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_766e368be42d8b18446b5ede","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":172,"at":"2026-04-16","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"258c2115516fcdec553478a14d7a8ca50f7cd834ae52dd464e6aa6e37c946bba","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-14","notes":null,"standard_error_points":3.2050494780000003,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Qwen3.6-35B-A3B","source_effort":null,"source_model_version":"qwen3.6-35b-a3b","source_row_date":"2026-04-16","source_scaffold":"little-coder","upstream_leaderboard_url":null},"run_count":null,"scaffold":"little-coder","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"6856a36d352fa394ee80a5ad93584c6639cde50d74dbd3dd23025f5c78133285","result":{"confidence_high":29.31560484208,"confidence_low":16.75181088832,"sample_count":89,"score":23.0337078652,"score_display":"23.0","source_stated_rank":166},"run_fingerprint":"c7c7768860cc68bb84db55ccaf13a79c84e0edf10fa3db7b1ff0086f85fa3050","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c97090d890f08c1934061a46","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"mean_reward","name":"Terminal-Bench 2.0","release_id":"release_445dddec34d8f59576432f47","split_or_window":"qwen-vendor-reported","unit":"percent","version":"2.0"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:24Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"fc1fa88844058ee1a82c975b625e82c358fada915ab2bd2dc5f6b02a854fdf11","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Benchmark table; terminal-bench-2; Qwen3.6-35B-A3B column"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Terminal-Bench 2.0; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"terminal agent toolset"},"run_count":null,"scaffold":"not reported","tools_allowed":"terminal agent toolset"},"protocol_fingerprint":"4f54fd48a37439f4c309fef796a8a535b52a10600a940a93d5c138cf29a69bba","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":51.5,"score_display":"51.5","source_stated_rank":null},"run_fingerprint":"83c6d3ee75b17e88775c294632ad77f0ca9deee8218e93ed2bb00fb6ef80b423","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Benchmark table; terminal-bench-2; Qwen3.6-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_771e537906e49bf2223bf52f","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e914420e7d3130760656a017edc44287b2ef63128849d29fc342d4bca5cad3f7","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-35b-a3b","upstream_model_label":"Qwen3.6 35B A3B (Reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"d629984b63a1592edd02d46128b00087193eb7abc1b8fa920e3228bf3824b5cc","result":{"confidence_high":55.273852,"confidence_low":35.032217,"sample_count":89,"score":44.94382,"score_display":"44.9","source_stated_rank":null},"run_fingerprint":"192e60ede67bd215d717ff998077c39f59d2833680f724555ef069e1deb3797a","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_fa62330ab4b31843f371c58b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7067dc9abf865bb267d17a905068a7fe04ceb397bdc985956379939c77f49f04","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-6-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.6 35B A3B (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"9fc0d2fb2732b2459cfa9d2a450a41c3063c04f613d5f6dfe6db5274d4acec9c","result":{"confidence_high":51.953085,"confidence_low":31.890365,"sample_count":89,"score":41.573034,"score_display":"41.6","source_stated_rank":null},"run_fingerprint":"966cf89628ba4ce2ddc19549949c6989d34592fb62039bb7a1abe5d67a8d6a6e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_rate","name":"τ³-Banking · Official harness","release_id":"release_c6e5abafbb7bffc0975932de","split_or_window":"qwen-official-user-model-bm25","unit":"percent","version":"TAU3-Bench · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"e574e87e5070501a934849b975a8bcea9b21372b779ddc0338d4274b5ad49904","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; General Agent; TAU3-Bench; Qwen3.6-35B-A3B column"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card TAU3-Bench evaluation using the official GPT-5.2-low user model and default BM25 retrieval; domain aggregation, task count, model reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"TAU3 domain tools"},"run_count":null,"scaffold":null,"tools_allowed":"TAU3 domain tools"},"protocol_fingerprint":"ac033ae84d9e8c3792065e58a44a15f3d80b846134d957b52b2b43ded2458ebb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.2,"score_display":"67.2","source_stated_rank":null},"run_fingerprint":"0eb52403dc1e2c99a89458965f90edefc52618f10f346ce6633a4e5bbf742904","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; General Agent; TAU3-Bench; Qwen3.6-35B-A3B column","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_771e537906e49bf2223bf52f","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"daebacc04fa2304d50b49bba05176f83dbc0a9cf1966305fa52a912458628533","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · tauBanking"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b","upstream_model_label":"Qwen3.6 35B A3B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"f679f65a36e510becea7f7cf694989f1759eba4314415d44912f14abbfa3cd37","result":{"confidence_high":16.70105,"confidence_low":4.958266,"sample_count":97,"score":9.278351,"score_display":"9.3","source_stated_rank":null},"run_fingerprint":"a1ab808f9c331959eda7913c0df787d5f5e8661b7feaa4297b59b02f96a970e0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_fa62330ab4b31843f371c58b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cf1a4756cd57c2e37b66757a8ae0e89c3cd36112089951ec0afa6fc95ebebaea","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-16","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · tauBanking"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-6-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.6 35B A3B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"6d75ae4b3147d95a0ebf08770f9e7902a03d045c0959120c0cb01c7b2346079d","result":{"confidence_high":11.77511,"confidence_low":2.347633,"sample_count":97,"score":5.360825,"score_display":"5.4","source_stated_rank":null},"run_fingerprint":"747a67ef0d3f30667a3435a3a526ef8314fefd04865b200853a5ba671f4fd623","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_5af9fe5a9d5e275323798c8b","split_or_window":"self-contained-78-no-api-subset","unit":"percent","version":"SkillsBench · Qwen 78-task self-contained subset"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":172,"at":"2026-04-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-16","status":"stale"},"measurement_id":"7c4f2d563f9e4f620d204582b31e1e91f63ef7c14f7cefb4c42bc5275d6c28eb","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 78 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.6-35B-A3B column; SkillsBench methodology note"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen vendor evaluation through OpenCode on 78 self-contained tasks, excluding API-dependent tasks; average of five runs; exact tool policy and reasoning mode not disclosed","question_count":78,"reasoning_effort":"reported","result_published_at":"2026-04-16","source_content_hash_method":"raw_bytes","source_content_sha256":"c4ddaa065649ff6352648f64747a16eda31726f3e34add94ce04abb461c77b75","source_published_at":"2026-04-16","tools":"not reported"},"run_count":5,"scaffold":"OpenCode","tools_allowed":"not reported"},"protocol_fingerprint":"1ef4df8bbd26617f6a97c4748bab4ad0b89cb62c839ee783c7a5b25a8ac69428","result":{"confidence_high":39.551323,"confidence_low":19.848298,"sample_count":78,"score":28.7,"score_display":"28.7","source_stated_rank":null},"run_fingerprint":"ac46754bd3952d481779be1a27c2f62f553c72965f6be0ea0f7234bf91c87341","source":{"content_hash":"ad790c81919e4560acf0f8f11011ef8c21748d81c86fd522c5ced0156885dc39","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","id":"qwen3-6-35b-a3b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-35B-A3B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.6-35B-A3B column; SkillsBench methodology note","name":"Qwen3.6-35B-A3B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_771e537906e49bf2223bf52f","provider_config":{"source_id":"qwen3-6-35b-a3b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"725e9f3dca19b6a470d667e37b531f21e8aaaa92118fed284728f4fee156fbee","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-16","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-6-35b-a3b","upstream_model_label":"Qwen3.6 35B A3B (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"0ca5d15094cd7563ba6c900da5c7e11dc4601a0c1290d6a9364bed370e1fde09","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":879.21,"score_display":"879","source_stated_rank":null},"run_fingerprint":"6bead62813315718026d84a62fe09965bd8b0124cbf12f4075df83a84add3b55","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_fa62330ab4b31843f371c58b","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7e428217c464efd283662a1ee1a80a27e3fc6d6a1bafb6fc2db262305158f696","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-16","reported_confidence_interval":"-29 / +29","source_locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-6-35b-a3b","name":"Qwen 3.6 35B-A3B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-6-35b-a3b-non-reasoning","upstream_model_label":"Qwen3.6 35B A3B (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"13fb51e58a41dd5b692bcb6fdd471679186e5b6f08f0a7769165fe12875dc511","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":837.35,"score_display":"837","source_stated_rank":null},"run_fingerprint":"aa37a96dfb76e7240690c7113895884267a33e0956461985d0a615ceb5dade5f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.6 35B A3B (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_c69dc5716abeec6f1e0c8be8","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"327fec757993581f08a92ada5903943f462834cfdec8f629cdc553e537498b28","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · Qwen3.8-Flash-Next · intelligenceIndex"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-flash-next","upstream_model_label":"Qwen3.8-Flash-Next"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"be9209420897969666964444dd2b6a50e5378f0081670d3cedbffa1955d69505","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.822302,"score_display":"39.8","source_stated_rank":null},"run_fingerprint":"079261ac6d56550f85b1b422bf748a4b1f7d7d8d79ee2cac1c70ff80c3aa12be","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8-Flash-Next · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a9a476e7573bba3097d22172","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"general","higher_is_better":true,"id":"livebench","metric":"category_balanced_average","name":"LiveBench","release_id":"release_c94a1cfb9d281df4254ff243","split_or_window":"overall-seven-category","unit":"percent","version":"2026-06-25"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":102,"at":"2026-06-25","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:36Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"11450f9003467fe564e8d965bce40ba7fba628dddcff1e3b2734d9b6c3890682","metric_details":{"category_scores":{"Agentic Coding":61.61633333333333,"Coding":72.5505,"Data Analysis":74.24,"IF":77.10849999999999,"Language":74.643,"Mathematics":85.821,"Reasoning":87.37975}},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a84af31207ac7d6490dcb1124920c18a5cc4c38ca783ab2559a81357f257ddaa","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.19415476190476,"score_display":"76.19","source_stated_rank":31},"run_fingerprint":"1cac16bc0c153fb2dad4d96eb6b2f9b963b2abb5c131ef3941052b176a30f512","source":{"content_hash":"8769411d022294e84ca28c83fcb0211469516273e5a760300315cebadd65474c","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://livebench.ai/","id":"livebench","last_fetched_at":"2026-10-05T12:33:29Z","license":"Upstream suite Apache-2.0; frontend score-feed license to review","locator":null,"name":"LiveBench","redistribution_policy":"aggregate-scores-with-attribution-review","result_url":"https://raw.githubusercontent.com/LiveBench/new-livebench/main/public/table_2026_06_25.csv"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d95c76a014239d533b5e8b3f","provider_config":{"source_id":"livebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution-review","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_287aa4bcb31b5b1ba26e562a","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":39,"at":"2026-08-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-27","status":"fresh"},"measurement_id":"49d0aee1d3d042d3f65fe4f1ee5742ec7a78f5123e770f6928e9284a4cc7696e","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Language benchmark table; General; Multidisciplinary reasoning HLE; Qwen3.8-Flash-Next column; HLE methodology note"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":"GPT-4o","run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE evaluation judged by GPT-4o; task count, split, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-27","source_content_hash_method":"raw_bytes","source_content_sha256":"35ca37ccc366f1ba478dab33841a2c0c18ce53fd62f291ca05341f7728b225b2","source_published_at":"2026-08-27","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"70e596e918ba6dfcb89e204cfc4f567f7cfee53e41c3d48d086272421582cf13","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.9,"score_display":"35.9","source_stated_rank":null},"run_fingerprint":"6d626bd77f15e5242b6d18759f61c6bfbf2db0fbd66caf747183fcf5f773065f","source":{"content_hash":"a8132c0d08dcb8936d1a9078fbf74a39072aa1d37bc0b3c112c58e2308514ae5","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next","id":"qwen3-8-flash-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen Community License model card; aggregate score facts with attribution","locator":"Language benchmark table; General; Multidisciplinary reasoning HLE; Qwen3.8-Flash-Next column; HLE methodology note","name":"Qwen3.8-Flash-Next official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0ee939ebaa5890202c8663ac","provider_config":{"source_id":"qwen3-8-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":40,"at":"2026-08-26","basis":"evaluation_at","evaluated_at":"2026-08-26","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9f4ccbf34bf0f43e380a4a34e26d638dfc792ef360ad79c5d02dd69c68dac5f9","metric_details":{"modality_lane":"not reported","notes":"Judged by GPT-4o per the card, not the task's default o3-mini grader.","num_parameters":179999981459,"pull_request":12,"result_file_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next/resolve/refs%2Fpr%2F12/.eval_results/Qwen3.8-Flash-Next.yaml","source":{"isExternal":false,"name":"Qwen3.8-Flash-Next model card","url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next"}},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":35.9,"score_display":"35.9","source_stated_rank":32},"run_fingerprint":"d53c69269e2264737b657fa087e1ca02e81d6dfdbc817a158dd1dafdc9051e22","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_17bff1e1526ad17f5595dc7c","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"86b692c0fde76a378d32a26949be73dafc8859469610cb4226920d152293f244","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · Qwen3.8-Flash-Next · hle"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-flash-next","upstream_model_label":"Qwen3.8-Flash-Next"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d53bf25b2613b7cf0c9a482c0732a0daab7f8ab285d01b922f5b88d2683b4c9b","result":{"confidence_high":40.112427,"confidence_low":36.019035,"sample_count":2158,"score":38.044486,"score_display":"38.0","source_stated_rank":null},"run_fingerprint":"fcb475c2bb03030fd5aed8d1d080d753792a207b3940774c5ab47826c2ec7067","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8-Flash-Next · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a9a476e7573bba3097d22172","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"58047205c246717989494bf3c98a03bd1e86cc7aa5ac757926f5024a08bb6b72","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · Qwen3.8-Flash-Next · gpqa"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-flash-next","upstream_model_label":"Qwen3.8-Flash-Next"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8c4882daafbe0afa4214c03860eace4d4df55c179275c19f4b9e8c3154da2249","result":{"confidence_high":95.277787,"confidence_low":87.757622,"sample_count":198,"score":92.323232,"score_display":"92.3","source_stated_rank":null},"run_fingerprint":"4ea6387d04a9884dbeec189dbb9d4c9d27a911bebdfb773337f65e2b53cb9973","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8-Flash-Next · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a9a476e7573bba3097d22172","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_beac9b3474dc330fed0ebc77","split_or_window":"diamond-vendor-protocol-undisclosed","unit":"percent","version":"GPQA Diamond · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":39,"at":"2026-08-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-27","status":"fresh"},"measurement_id":"51149684f58b4d803c11a837006b1ec8baf3fbd91446e6a85ef7bbbd01625e11","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Language benchmark table; General; Scientific reasoning GPQA Diamond; Qwen3.8-Flash-Next column"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card GPQA Diamond evaluation; prompt, shots, tool policy, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-27","source_content_hash_method":"raw_bytes","source_content_sha256":"35ca37ccc366f1ba478dab33841a2c0c18ce53fd62f291ca05341f7728b225b2","source_published_at":"2026-08-27","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"72ae89ce870144e0cbda858a578a242a80ff205968afa7136a92efb6ba47e233","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.7,"score_display":"91.7","source_stated_rank":null},"run_fingerprint":"046c755985633c5dfc37793aedf2909767ec17ca39542789769c706c5f28d977","source":{"content_hash":"a8132c0d08dcb8936d1a9078fbf74a39072aa1d37bc0b3c112c58e2308514ae5","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next","id":"qwen3-8-flash-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen Community License model card; aggregate score facts with attribution","locator":"Language benchmark table; General; Scientific reasoning GPQA Diamond; Qwen3.8-Flash-Next column","name":"Qwen3.8-Flash-Next official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0ee939ebaa5890202c8663ac","provider_config":{"source_id":"qwen3-8-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f087157a209e7045683e094764ed9384c177b8faa1d2468f0d24836a6b962981","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · Qwen3.8-Flash-Next · mmmuPro"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-flash-next","upstream_model_label":"Qwen3.8-Flash-Next"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"14ceb8d686bf972ead6de50bb171aa1099b977e38823bbac90c2c582e84075e3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.768786,"score_display":"79.8","source_stated_rank":null},"run_fingerprint":"812cbb74a5b7456d8522ffc1911510fd4c1c5f96cac92295be337fb4f4dd6255","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8-Flash-Next · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a9a476e7573bba3097d22172","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d4a1724e0ae9eb4cdcff6049c58dc575770b6d8ba81c0cc3ea1ca390ed9d103b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · Qwen3.8-Flash-Next · critpt"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-flash-next","upstream_model_label":"Qwen3.8-Flash-Next"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"a9b28fa84c7b968eff98d679e5d611d0b2741afa68a0afe1b59a1d4001e17804","result":{"confidence_high":20.620776,"confidence_low":5.708014,"sample_count":70,"score":11.142857,"score_display":"11.1","source_stated_rank":null},"run_fingerprint":"4f8f602b3019d9010140660205581d790997ac34c73b8b0f8c00c1a9596b05d7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8-Flash-Next · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a9a476e7573bba3097d22172","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"deepswe","metric":"resolved_rate","name":"DeepSWE","release_id":"release_12e8a62d68d933c29dcdf020","split_or_window":"vendor-model-card-run","unit":"percent","version":"DeepSWE 1.1 vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":39,"at":"2026-08-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-27","status":"fresh"},"measurement_id":"e90a3198f6d981321746ff06de256d780783c636f766d15c1c8338fa409b1cdb","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Evaluation table row Qwen3.8-Flash-Next; DeepSWE 1.1 column; evaluation details immediately below table"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card vendor run on DeepSWE 1.1; best of Claude Code and mini-SWE-agent; temperature 1.0; top_p 0.95; 256K context; reasoning effort not disclosed","reasoning_effort":"reported","result_published_at":"2026-08-27","source_content_hash_method":"raw_bytes","source_content_sha256":"35ca37ccc366f1ba478dab33841a2c0c18ce53fd62f291ca05341f7728b225b2","source_published_at":"2026-08-27","tools":"Claude Code and mini-SWE-agent tool environments"},"run_count":null,"scaffold":"best of Claude Code and mini-SWE-agent","tools_allowed":"Claude Code and mini-SWE-agent tool environments"},"protocol_fingerprint":"8952100b315bc05cb6845e1f3c7939f39264ad2d12eab342c072a06b1c4f1351","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":58.7,"score_display":"58.7","source_stated_rank":null},"run_fingerprint":"949a28e6130f4052403eb43c6cdcdf4c1b49a73e88bd5103937d5640882024fc","source":{"content_hash":"a8132c0d08dcb8936d1a9078fbf74a39072aa1d37bc0b3c112c58e2308514ae5","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next","id":"qwen3-8-flash-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen Community License model card; aggregate score facts with attribution","locator":"Evaluation table row Qwen3.8-Flash-Next; DeepSWE 1.1 column; evaluation details immediately below table","name":"Qwen3.8-Flash-Next official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0ee939ebaa5890202c8663ac","provider_config":{"source_id":"qwen3-8-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_d4a5abde3ed9874198f1dc1f","split_or_window":"qwen-refined-vendor-run","unit":"percent","version":"SWE-bench Pro · Qwen refined vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":39,"at":"2026-08-27","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-08-27","status":"fresh"},"measurement_id":"b5dbc17c381a082671954efda9c73a4ff86e85c3b925260b9d1cdb6d96f0c7f3","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Evaluation table row Qwen3.8-Flash-Next; SWE-Pro column; evaluation details immediately below table"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card vendor run on SWE-bench Pro with documented corrections/refinements to problematic tasks; task count and reasoning effort not disclosed; temperature 1.0; top_p 0.95; 256K context","reasoning_effort":"reported","result_published_at":"2026-08-27","source_content_hash_method":"raw_bytes","source_content_sha256":"35ca37ccc366f1ba478dab33841a2c0c18ce53fd62f291ca05341f7728b225b2","source_published_at":"2026-08-27","tools":"Claude Code tool environment"},"run_count":null,"scaffold":"Claude Code","tools_allowed":"Claude Code tool environment"},"protocol_fingerprint":"e9613eb19beed5b6a0cbc6aeecaf8a370ac11ff2cedd0346a73a16687b68014b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":62.5,"score_display":"62.5","source_stated_rank":null},"run_fingerprint":"54a95b94303aebf3ceb17723ab72125ddb799c6e56319e08f085edcf2244f071","source":{"content_hash":"a8132c0d08dcb8936d1a9078fbf74a39072aa1d37bc0b3c112c58e2308514ae5","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-03T09:21:26Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next","id":"qwen3-8-flash-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Qwen Community License model card; aggregate score facts with attribution","locator":"Evaluation table row Qwen3.8-Flash-Next; SWE-Pro column; evaluation details immediately below table","name":"Qwen3.8-Flash-Next official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.8-Flash-Next"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0ee939ebaa5890202c8663ac","provider_config":{"source_id":"qwen3-8-flash-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"93ff1b616e9f6c36b1e22551f6a77704bfe7e82c6666398c94823c885aec1443","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · Qwen3.8-Flash-Next · scicode"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-flash-next","upstream_model_label":"Qwen3.8-Flash-Next"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"0279b088c1ab9ac3ad72e0298d6dde02f87fe09e59c3565929d2d9b3bc461535","result":{"confidence_high":56.30728,"confidence_low":44.834893,"sample_count":288,"score":50.578704,"score_display":"50.6","source_stated_rank":null},"run_fingerprint":"d869eecf6ecd4ffc1315df2642fbad4af48bef48bc44ed04b87985522bf180ce","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8-Flash-Next · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a9a476e7573bba3097d22172","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef76ee4983fab76f76b44180be21ec1990e089f4ae5693b15fb4dacab418c24c","metric_details":{"license":"qwen-community-1.0","variance":20.21166783289356},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.8-flash-next","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"fdbc0bce796b9cc3cce59b935ec615298d8fab0b1b9bd38c492768ed2eb1d0a0","result":{"confidence_high":1646.3062761656715,"confidence_low":1628.6833036947621,"sample_count":6193,"score":1637.4947899302167,"score_display":"1637","source_stated_rank":16},"run_fingerprint":"b961c63889ccfd7593644df141b3d704a3f6b7f23ad373ffa92095d65aadfeeb","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c9c41d02d7ef3f31e9d42eeb","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"742bf1c840382f6471adb61755f81808f19f72f7a4dab39f608528b90c1fb2cc","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.8-Flash-Next · terminalbenchV21"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-8-flash-next","upstream_model_label":"Qwen3.8-Flash-Next"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"dbfba9fe1f58e51615d9ef3195b743889662d3640e50a42505c40437fa587b9c","result":{"confidence_high":91.832281,"confidence_low":77.461369,"sample_count":89,"score":86.142322,"score_display":"86.1","source_stated_rank":null},"run_fingerprint":"33eb476dc155c7079322d2b8db783182423854feeab298c629362ae1891f112e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8-Flash-Next · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a9a476e7573bba3097d22172","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b5e70b98fe9ca1c12b761da2d7509e904163194a1142647ce75269ad80def6a4","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","source_locator":"Leaderboard models table · Qwen3.8-Flash-Next · tauBanking"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-8-flash-next","upstream_model_label":"Qwen3.8-Flash-Next"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"227658af8f47a1f6a816c76d569d304c1d319e1ac96e4aebae63356abbe376f5","result":{"confidence_high":55.256086,"confidence_low":35.819026,"sample_count":97,"score":45.360825,"score_display":"45.4","source_stated_rank":null},"run_fingerprint":"ce23cc25094c1c360013cf7feb776102c2bf81f1866621320946797f42a8a30b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8-Flash-Next · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a9a476e7573bba3097d22172","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d08bbd9e82aef9ac573af8893492bfcc4af0345b16c0e379508a1bf306cf10a7","metric_details":{"confidence_level":0.95,"license":"qwen-community-1.0","session_count":83198},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Qwen3.8 Flash Next","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"af3c16d7efa7f1241abef4a65ad7c67c5ecc84ada3a22a065e88769560eade34","result":{"confidence_high":-0.0009772035192713267,"confidence_low":-0.013552000993865224,"sample_count":11249221,"score":-0.0072646022565682755,"score_display":"-0.0073","source_stated_rank":34},"run_fingerprint":"a5667d4f23bceed12accac1ba9d645285e97a3f3ec5f1bb3bc3799e1f99de47d","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_905688dc095b693c8f3fab53","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b0e0b63bb837283023704e2bab9ab876d08e569cd5aeef7a37b0718f8c5c7523","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-08-26","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Qwen3.8-Flash-Next · gdpval"},"model":{"id":"alibaba/qwen3-8-flash-next","name":"Qwen 3.8 Flash Next","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-8-flash-next","upstream_model_label":"Qwen3.8-Flash-Next"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"b3580ea38cb429fcaca0ecd8c76a50620b0b6a6c704047c4f5e6335ba1fcdc60","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1612.13,"score_display":"1612","source_stated_rank":null},"run_fingerprint":"f86fe6d5c8ef768ead9ec77d6ce62c1f6b0a804045c9f665905b1cd76ae1863d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.8-Flash-Next · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a9a476e7573bba3097d22172","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3a97020f34206fbe2d538d77d8934794a73e8334d5a6d1b9d42498311f45f4f4","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-02"},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemma 4 31B IT; model release: 2026-04-02","source_accessibility":"Open weights (restricted use)","source_model_name":"Gemma 4 31B IT","source_model_release_date":"2026-04-02","source_model_versions":["gemma-4-31b-it","gemma-4-31b-it_minimal"],"source_reasoning_efforts":["minimal"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"437703a487bfc9f1c0f64dd66009782e5cf7eb2b9f0be22d561884229f27fe97","result":{"confidence_high":144.94,"confidence_low":140.27,"sample_count":null,"score":142.75,"score_display":"142.75","source_stated_rank":95},"run_fingerprint":"c51d849e7cfa7219bc764814d3651195ed59771b96898a0b1ad3535a20272050","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5df12bd86522bd0fba242adc","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_9b8cd6ba0a835eb2da22bb93","split_or_window":"google-vendor-reported","unit":"percent","version":"HLE"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T20:06:26Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"33b6281ff3e4ea642abf9498634127dc6495925dfb92c55ccb9411fd7123b2a7","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-31B-it/resolve/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md","evidence_sha256":"c1ec2c93e2135e13d82a840a8daf7692444fe7f551fe011da4a8df9343347b25","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-31B-it","version_id":"bf7cb8664e132f451ecfdd500f9f0fae778962ee673de052a4de6a5fb9349a36"},"source_locator":"Benchmark Results; HLE no tools row; Gemma 4 31B IT column"},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"HLE no tools; Google model-card evaluation. Exact task revision, sample count, and per-benchmark reasoning settings undisclosed; supplemental provider evidence, not a canonical organizer score.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"e209f5c435b1e66055bd6d7c29185103ae8d2027a03a96b359335154f206b82b","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"9bf33a886a0f23c36ca21f71357edb46598a0aa53c037d5b817890035204e13e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":19.5,"score_display":"19.5","source_stated_rank":null},"run_fingerprint":"f29f4178978b7af215f39660737eae7e7c3029f1ce53d84b4ac3f50fbb003879","source":{"content_hash":"97a2485f13fd873b65d95d592d06719bca84ae9de163ea58a301eeaa5ff3282d","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-31B-it","id":"gemma4-31b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark Results; HLE no tools row; Gemma 4 31B IT column","name":"Gemma 4 31B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-31B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e32b67ae6bf59f0434d4b67f","provider_config":{"source_id":"gemma4-31b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:30:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:30:07.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d17d521b8876974906cdf8b273329e3b795c5673b90b83cc6d5b713e4a13eb15","metric_details":{"best_score_across_scorers":"0.104","model_release_date":"2026-04-02","stderr":0.9658016218524242},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"D5cbXacPnJ8akq2jAtifEQ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FD5cbXacPnJ8akq2jAtifEQ.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/D5cbXacPnJ8akq2jAtifEQ.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"3ac8135ce62560dc2e1a768e0d30d97275fa17858342f1639436428e2c14b10c","result":{"confidence_high":12.292971178830753,"confidence_low":8.507028821169248,"sample_count":1000,"score":10.4,"score_display":"10.4","source_stated_rank":84},"run_fingerprint":"23a7498ff19a417c932ce3d03e0e14e192896c001f3436f97abe1d9dbf90d212","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3bc4445984e2cfe1b5212307","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T23:59:38.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:59:38.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"914dd27fb0387ec7e1ddf224a84271498561adf901570ab8da951d676f7a4fd9","metric_details":{"best_score_across_scorers":"0.7575757575757576","model_release_date":"2026-04-02","stderr":3.053289223393202},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"P5L8qbXo42vTvbU3UvTNjk","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FP5L8qbXo42vTvbU3UvTNjk.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/P5L8qbXo42vTvbU3UvTNjk.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"32b3f877f64442a1212d93dcbfb81f5c80369c109f440911702295426a25237a","result":{"confidence_high":81.74202263542642,"confidence_low":69.77312887972508,"sample_count":null,"score":75.75757575757575,"score_display":"75.8","source_stated_rank":152},"run_fingerprint":"c81a98cbe4452361f09f627256e5a8b0baa5229b5e7de34b744d7221b1ce7119","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_af2ff99bab99820d4a9eb678","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_fa97cbc1bde3163225d15751","split_or_window":"google-vendor-reported","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"b85b577b3966622b313cfc74159dd61d7b12a871fa8d8de03dd8e7cce91884ad","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-31B-it/resolve/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md","evidence_sha256":"c1ec2c93e2135e13d82a840a8daf7692444fe7f551fe011da4a8df9343347b25","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-31B-it","version_id":"bf7cb8664e132f451ecfdd500f9f0fae778962ee673de052a4de6a5fb9349a36"},"source_locator":"Benchmark table; gpqa-diamond; Gemma 4 31B IT column"},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"GPQA Diamond; official Google model-card run with thinking. Exact task snapshot, repetitions and inference budget undisclosed.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"e209f5c435b1e66055bd6d7c29185103ae8d2027a03a96b359335154f206b82b","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"e4e3a552f788fcd515a2383fa14c70acd167165a5f43049da912a0588b01111a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":84.3,"score_display":"84.3","source_stated_rank":null},"run_fingerprint":"899512d78abe3302633197550458d03f173a3c5e05e45d248418eccb724b0f5f","source":{"content_hash":"97a2485f13fd873b65d95d592d06719bca84ae9de163ea58a301eeaa5ff3282d","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-31B-it","id":"gemma4-31b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; gpqa-diamond; Gemma 4 31B IT column","name":"Gemma 4 31B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-31B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e32b67ae6bf59f0434d4b67f","provider_config":{"source_id":"gemma4-31b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_cbef64e634e73d9217bfe565","split_or_window":"google-vendor-reported","unit":"percent","version":"MMLU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"4f9949cda683889721cbde50c847a62adada4351046b79163a263f6f7c97fe14","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-31B-it/resolve/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md","evidence_sha256":"c1ec2c93e2135e13d82a840a8daf7692444fe7f551fe011da4a8df9343347b25","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-31B-it","version_id":"bf7cb8664e132f451ecfdd500f9f0fae778962ee673de052a4de6a5fb9349a36"},"source_locator":"Benchmark table; mmlu-pro; Gemma 4 31B IT column"},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"MMLU Pro; official Google model-card run with thinking. Exact task snapshot, repetitions and inference budget undisclosed.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"e209f5c435b1e66055bd6d7c29185103ae8d2027a03a96b359335154f206b82b","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"78530b6152fd4eaecf30afb5b591e4d005694e92002f866fb1149eea1731e217","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":85.2,"score_display":"85.2","source_stated_rank":null},"run_fingerprint":"47a3f272644a74934efee821a31ac6b49d38ab79f6db5c4b81a7d28d95df49e4","source":{"content_hash":"97a2485f13fd873b65d95d592d06719bca84ae9de163ea58a301eeaa5ff3282d","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-31B-it","id":"gemma4-31b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; mmlu-pro; Gemma 4 31B IT column","name":"Gemma 4 31B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-31B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e32b67ae6bf59f0434d4b67f","provider_config":{"source_id":"gemma4-31b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T23:59:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:59:45.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e6ffaa443ba451c1cd41f7e8620144998a2cb49c0f57e03d72baf2e89982eddd","metric_details":{"best_score_across_scorers":"0.7333333333333333","model_release_date":"2026-04-02","stderr":6.666666666666667},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"jFJucVunfLgoQqoFvPVQWP","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjFJucVunfLgoQqoFvPVQWP.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/jFJucVunfLgoQqoFvPVQWP.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"07e5cbaa45798df58d2463abaff06d5b328765e707e6e30a522c548f450919c7","result":{"confidence_high":86.39999999999999,"confidence_low":60.266666666666666,"sample_count":45,"score":73.33333333333333,"score_display":"73.3","source_stated_rank":126},"run_fingerprint":"e0857fedf23dc738d58246268aeebcbdbc17964a08619ab576debb108f6cb0e3","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_af2ff99bab99820d4a9eb678","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_6f3fd0029277bfcad78a10b3","split_or_window":"google-vendor-reported","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T20:06:26Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"b79e7440839dbcf84a17ad59d2677f7937dc77a425fcc4325cca5892dd93c776","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-31B-it/resolve/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md","evidence_sha256":"c1ec2c93e2135e13d82a840a8daf7692444fe7f551fe011da4a8df9343347b25","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-31B-it","version_id":"bf7cb8664e132f451ecfdd500f9f0fae778962ee673de052a4de6a5fb9349a36"},"source_locator":"Benchmark Results; MMMU Pro row; Gemma 4 31B IT column"},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"MMMU Pro; Google model-card evaluation. Exact task revision, sample count, and per-benchmark reasoning settings undisclosed; supplemental provider evidence, not a canonical organizer score.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"e209f5c435b1e66055bd6d7c29185103ae8d2027a03a96b359335154f206b82b","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"b4ea8f2c161f678a1cd9587ed3df25a8dd6a87cbc1e1c751bec680b93db62fb0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.9,"score_display":"76.9","source_stated_rank":null},"run_fingerprint":"6f348d5593d6801efe9aeea7e579c7a2ddb03d14b3b9a7f12e2f1b68535d0b4d","source":{"content_hash":"97a2485f13fd873b65d95d592d06719bca84ae9de163ea58a301eeaa5ff3282d","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-31B-it","id":"gemma4-31b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark Results; MMMU Pro row; Gemma 4 31B IT column","name":"Gemma 4 31B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-31B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e32b67ae6bf59f0434d4b67f","provider_config":{"source_id":"gemma4-31b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a05f9779b39e3532c3823d405d597699b9568cddb644645609f0ff206f7b2d56","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-02","notes":null,"upstream_source_url":null},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemma 4 31B (Reasoning)","source_effort":null,"source_model_version":"gemma-4-31b-it","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"0cfe56e5bc37e596794d50564f26128c19f9853fed0f910971fbed831d41d643","result":{"confidence_high":7.588324,"confidence_low":0.255137,"sample_count":71,"score":1.42857142857143,"score_display":"1.4","source_stated_rank":136},"run_fingerprint":"d6639c1fe1c1db41c82bbc23f571da5dc5ed8925a2e51b3905658d7f77abd477","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7d82a165512b034548c396d2","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T23:59:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:59:50.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c738382e7ff2242ba9775f1c20497edfec6c88613a931e35779326353936bd33","metric_details":{"best_score_across_scorers":"0.05","model_release_date":"2026-04-02","stderr":2.1904291355759034},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4LjmaK4MRrxuDXUiZsvA3o","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4LjmaK4MRrxuDXUiZsvA3o.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4LjmaK4MRrxuDXUiZsvA3o.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"628ae5a928e2ad2fe4addf273d01f374c73914deb9ca2aaf8844989f64aee534","result":{"confidence_high":9.293241105728772,"confidence_low":0.7067588942712293,"sample_count":100,"score":5.0,"score_display":"5.0","source_stated_rank":161},"run_fingerprint":"af511e9a098941b059998009d6de5f680da03c08bc01759db88f1cb7e2ab0fb0","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_af2ff99bab99820d4a9eb678","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_1d1cf16bb310d2fb0576bb02","split_or_window":"google-vendor-reported","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"2f34f468b18cbd0c11832768d6dd2e9ea628350f6de035ca9b3dfe45f767d6da","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-31B-it/resolve/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md","evidence_sha256":"c1ec2c93e2135e13d82a840a8daf7692444fe7f551fe011da4a8df9343347b25","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-31B-it","version_id":"bf7cb8664e132f451ecfdd500f9f0fae778962ee673de052a4de6a5fb9349a36"},"source_locator":"Benchmark table; livecodebench; Gemma 4 31B IT column"},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"LiveCodeBench v6; official Google model-card run with thinking. Exact task snapshot, repetitions and inference budget undisclosed.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"e209f5c435b1e66055bd6d7c29185103ae8d2027a03a96b359335154f206b82b","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6d83bbcf136942f9b5279b1d5eef6f61d1c43f1bfd5d2d2a9b7aee53f4a643f4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":80.0,"score_display":"80.0","source_stated_rank":null},"run_fingerprint":"eaf6d59bbcdd16bcaac3a5ac06f41977df3acc8aeadd3ee2df9e26b99c1817a6","source":{"content_hash":"97a2485f13fd873b65d95d592d06719bca84ae9de163ea58a301eeaa5ff3282d","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-31B-it","id":"gemma4-31b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; livecodebench; Gemma 4 31B IT column","name":"Gemma 4 31B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-31B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e32b67ae6bf59f0434d4b67f","provider_config":{"source_id":"gemma4-31b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"389e3c9919f0a7f55ec4cdff8bbfe3f060cd96bcbad87b24608144c3a3d1c02a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-02","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemma 4 31B (Reasoning)","source_effort":null,"source_model_version":"gemma-4-31b-it","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"78047ad15e5eb077962e61f38f7459e1eec7493ed9d02e1063369fc7aa5a19be","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.4027777777778,"score_display":"43.4","source_stated_rank":129},"run_fingerprint":"a78ee7e7895d5e754b14de745234a199c437d03f6646675f98fcf4d66e60929c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e139c296e22aa94481af21b8","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3d60158b482193a599dc62106d4c53ef6537586fa6402263fd23eb2f64ae52d9","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.00475","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-02","notes":null,"upstream_source_url":null},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemma-4-31b-it","source_effort":null,"source_model_version":"gemma-4-31b-it","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"7ad5ec9cfe49e1d8ab07f2b42696a1b26507dbdcb90832dad73af780b7a383e0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":925.5,"score_display":"925.50","source_stated_rank":56},"run_fingerprint":"e56cafdec6a9aba1b306eef03d98ea626bd07e3e20f523d1275cd3b97acad946","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9bb470d85c107daac9f268e7","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"7019e1a2070d14e672beea6d08dee8baed8c19e1dbcd903ea84258baf495c7a0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-02","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemma-4-31b-it","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"a1493a09d9576bf0250a9a2dc1b687cc35f4f06bd586fe3e0007f0ff135aad04","result":{"confidence_high":1371.18,"confidence_low":1357.08,"sample_count":10139,"score":1364.13,"score_display":"1364.13","source_stated_rank":99},"run_fingerprint":"521891f6d7be5dcacfa96296b0925096d3e5a0899513004539e2153e1f59f95f","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d9228fbf3d63a4e480591aec","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ca932566104c5f5e00664177eeb28be97bb631ea578967cabf7ff9ed39e2071a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-04-02","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemma-4-31b-it","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"18ea39e9a2f389d9640f329d7b2fc41406f5d7e4329aa71c69b1482d9dbd6ece","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.26,"score_display":"52.3","source_stated_rank":67},"run_fingerprint":"5645e0e0a60d79fc4ae3967a57116c9f44879a6da0c3d4e6202467716b43d101","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e9bf89560c783f7a74d891aa","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifbench","metric":"inst_level_strict_acc","name":"IFBench","release_id":"release_4adf348ac2f76539ed7ccb10","split_or_window":"ifbench-test-300-prompts-344-instructions","unit":"percent","version":"swallow-evaluation-instruct"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-04T14:29:18Z","observed_at":"2026-10-04T14:29:18Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fd2bcf07c727dcc8176a3dcd5cf4a9114db31356efb3bec74b4daed4945ffc16","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","instruction_count":344,"model_release_date":"2026-04-02","prompt_count":300},"model":{"id":"google/gemma-4-31b-it","name":"Gemma 4 31B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"rule-based instruction checkers (lighteval 0.11.0 with Swallow fixes)","run_config":{"decoding":"temperature 0.6, top-p 0.95","leaderboard_revision":"c58a13a852eba061d639af7caf4ddbdff33b36e3","protocol":"Swallow LLM Leaderboard: swallow-evaluation-instruct (lighteval), allenai/IFBench_test, instruction-level strict accuracy; the final answer without the thinking part","swallow_model_id":"google/gemma-4-31B-it","swallow_reasoning":"on"},"run_count":null,"scaffold":"swallow-evaluation-instruct (lighteval), swallow|ifbench_singleturn","tools_allowed":"none"},"protocol_fingerprint":"6cfb052f9166158b3e6c3af96acd982cbea69c72aac5685f77a8a96d272bc4cb","result":{"confidence_high":null,"confidence_low":null,"sample_count":300,"score":76.5,"score_display":"76.5","source_stated_rank":null},"run_fingerprint":"6bc1a87a33305bba5e345913c69faa169b17aca9640083ab6484f16524d31678","source":{"content_hash":"fffdc5bb4b3de214a8a4ce885cf286e13a4b1488d9a4a0791e5918d1918b74cb","fetched_at":"2026-10-04T14:29:18Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://swallow-llm.github.io/leaderboard/index-post.en.html","id":"swallow-llm-leaderboard","last_fetched_at":"2026-10-04T14:29:18Z","license":"Leaderboard content CC-BY-4.0 (Swallow LLM Team, README of swallow-llm/leaderboard)","locator":null,"name":"Swallow LLM Leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/swallow-llm/leaderboard/c58a13a852eba061d639af7caf4ddbdff33b36e3/swallow_leaderboard_post_en.js"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_82d7b860b7defe3c83636cc3","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"aa6f8f4fa34fa13340dff39ad1b1450c0808f741d1e868abfcad223d4950a094","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-24"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen3.5-9B; model release: 2026-02-24","source_accessibility":"Open weights (restricted use)","source_model_name":"Qwen3.5-9B","source_model_release_date":"2026-02-24","source_model_versions":["qwen3.5-9B","qwen3.5-9B_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"c1a0efaa140886ca2e8fc09a18482cb82c4c720b156e97b94476a821a718c700","result":{"confidence_high":141.33,"confidence_low":136.45,"sample_count":null,"score":139.46,"score_display":"139.46","source_stated_rank":117},"run_fingerprint":"84fd0804ba60c3df55cd833615242f8fb4c8142748833b931707129d6b8dbe04","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_92d32148ceeb2d09e13b1a18","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"919a28d102819a327dc2bd76e1cfece638cc5b1a2c60081e3b10f9c946391a04","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-03-02","source_locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b","upstream_model_label":"Qwen3.5 9B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"013b495d25f19a54698bffd984c8f850cfd4957e76248c07a8585ce70c19eade","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":11.196165,"score_display":"11.2","source_stated_rank":null},"run_fingerprint":"ecec315db789f8742a94a807b7417cf72fb05116f2b9b4471f5ce7a3ffe275f8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e4151cd1887053dc66ccba8341c1e9c4916ffc7ac66a902093eda16a70596f45","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b-non-reasoning","upstream_model_label":"Qwen3.5 9B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"39051ee0cbc76614814ffdd50152783af5b8e7e9c81d1af2f215c23fa0f835bc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.265996,"score_display":"13.3","source_stated_rank":null},"run_fingerprint":"654b9fabb8fc6c2332c291bfcfec252ca0418558876c14146160b2249670479d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_4779204b36659935aea6f3dc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eac60ecfbc5304acb840e6534901ba796619833317a3d05340d49b5ea0d3d305","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-03-02","source_locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · hle"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b","upstream_model_label":"Qwen3.5 9B (Reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a43aeaa6c5f4025206197d00ccb588fb5038c3c6347cb2bc4aa6451735372583","result":{"confidence_high":16.486806,"confidence_low":13.48031,"sample_count":2158,"score":14.921223,"score_display":"14.9","source_stated_rank":null},"run_fingerprint":"1e688e4682248eb294307182c84764acd5f600f991364d5102934947627a4778","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"01d71bd48fd4e7654ca4ab65f5105296389e0a405151dd85434aedc5613dac9f","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · hle"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b-non-reasoning","upstream_model_label":"Qwen3.5 9B (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f90d92ab275e0da9fba7c26a0f56ae1f2a970376d82877337128a9f3ac2c2fbf","result":{"confidence_high":10.662729,"confidence_low":8.202742,"sample_count":2158,"score":9.360519,"score_display":"9.4","source_stated_rank":null},"run_fingerprint":"851b4cf613ae7b6c29e23f3e2e4dccfd883ea8a8dc2c09377c3ddd5abd74004a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_4779204b36659935aea6f3dc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"769be2cac0fb20a1e7ba2e6fa808259f019dce556f4cbbc17f9aa4fb71776189","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-03-02","source_locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b","upstream_model_label":"Qwen3.5 9B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f5db9edfb19b0f43f7162ff11caa7afae80faee65882151e87c57fea3782598e","result":{"confidence_high":85.509219,"confidence_low":74.537869,"sample_count":198,"score":80.606061,"score_display":"80.6","source_stated_rank":null},"run_fingerprint":"2f331cc292cbad045bf145176a2c72b883e2134bb3632803040fe32d9e0590aa","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e69492e1ed5eb69d47328515aca93e3e771aeab9034d6d68f39ab300bdceea72","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b-non-reasoning","upstream_model_label":"Qwen3.5 9B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a4e3a3267cf0d7f4577946bc974e5bd12ac8d571212c1fe0c8db15cb4a2d2bdc","result":{"confidence_high":83.727321,"confidence_low":72.356262,"sample_count":198,"score":78.585859,"score_display":"78.6","source_stated_rank":null},"run_fingerprint":"4fadf36aa09aadcb80edc5f330f3bb04b7fda6689fd9c7e51bab6ad15ea19eaa","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_4779204b36659935aea6f3dc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":38,"at":"2026-08-27T14:30:46.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T14:30:46.000Z","first_observed_at":"2026-08-28T00:52:16Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3393620d8d9c188630e302512af2e44c99fc7a0ebd1d411035063b4d5a962443","metric_details":{"best_score_across_scorers":"0.7891414141414141","model_release_date":"2026-02-24","stderr":2.388848562991196},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"AczFCvkWR5SmdbV7eXruqj","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAczFCvkWR5SmdbV7eXruqj.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/AczFCvkWR5SmdbV7eXruqj.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"a3fd9c0907e4787e325fb985ba790e8d5d1abd996ef707ea2e0f5475a4f60749","result":{"confidence_high":83.59628459760415,"confidence_low":74.23199823067867,"sample_count":null,"score":78.91414141414141,"score_display":"78.9","source_stated_rank":132},"run_fingerprint":"f94acd393561228cb72e44760e8cd4ed4e6d0cabe42e39644a3684cf64759188","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_4779204b36659935aea6f3dc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":38,"at":"2026-08-27T14:30:17.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T14:30:17.000Z","first_observed_at":"2026-08-28T00:52:16Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cfd8db774e1cc862355bb35b53bc16c022053e28c4aa13a09c7d827cccb639d0","metric_details":{"best_score_across_scorers":"0.7897727272727273","model_release_date":"2026-02-24","stderr":2.365568030958479},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"o6migB5WP27uLr9nUWpCiM","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fo6migB5WP27uLr9nUWpCiM.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/o6migB5WP27uLr9nUWpCiM.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"3f04e54afb7d6e1be3f8de204e070c8c567f3067b435a81eb599e5a8b3c9f8fe","result":{"confidence_high":83.61378606795135,"confidence_low":74.34075938659412,"sample_count":null,"score":78.97727272727273,"score_display":"79.0","source_stated_rank":131},"run_fingerprint":"5ac1f374d746e44c683e419b5ad42f18acb43bc6f5b813c6ab3bd10e060e2c90","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e03cd3d99c8f35b707a934fa","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_4840516eea283f02fbe24ea5","split_or_window":"qwen-vendor-reported","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":217,"at":"2026-03-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-03-02","status":"stale"},"measurement_id":"4b9311e3a251a96f0fff9726e8482d78aa27d7e7200d53430c14ca3a9c4ba8be","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/Qwen/Qwen3.5-9B/resolve/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md","evidence_sha256":"a1d07378ec333f3f6ff04e86aaaf4060e8cf3e1959123eb17fecb19e76dd0816","kind":"reviewed-static-report","published_at":"2026-03-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/Qwen/Qwen3.5-9B","version_id":"879c44ca5a6615044e90c0fad04eace605ab79c49db22a8f1684fafa29b2a0b4"},"source_locator":"Benchmark table; gpqa-diamond; Qwen3.5-9B column"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"GPQA Diamond; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-03-02","source_content_hash_method":"raw_bytes","source_content_sha256":"c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0","source_published_at":"2026-03-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"12e271f0b7109f7a6ce41f2794175b5b030b516621953d2f57f94eaa5c2181e5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":81.7,"score_display":"81.7","source_stated_rank":null},"run_fingerprint":"518d12e8a08a9f63bb82944754b0fe5cb5a0cdb505c8d2e95af1bd4643ea9d11","source":{"content_hash":"cd4cf4de3bf8c568c9c43f8211abbf4c12e41b290a91fce5f8fce5ee80023b44","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:25Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-9B","id":"qwen3-5-9b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; gpqa-diamond; Qwen3.5-9B column","name":"Qwen3.5-9B official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-9B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b87cc44f5a3902b1b71e87ed","provider_config":{"source_id":"qwen3-5-9b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_cd90aeaaf8e38ae0a7eb7edd","split_or_window":"qwen-vendor-reported","unit":"percent","version":"MMLU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":217,"at":"2026-03-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-03-02","status":"stale"},"measurement_id":"9e3c9648fd59b167e1c8c0819cebc876bd729996c0c8cd50f46b075e45062476","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/Qwen/Qwen3.5-9B/resolve/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md","evidence_sha256":"a1d07378ec333f3f6ff04e86aaaf4060e8cf3e1959123eb17fecb19e76dd0816","kind":"reviewed-static-report","published_at":"2026-03-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/Qwen/Qwen3.5-9B","version_id":"879c44ca5a6615044e90c0fad04eace605ab79c49db22a8f1684fafa29b2a0b4"},"source_locator":"Benchmark table; mmlu-pro; Qwen3.5-9B column"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"MMLU-Pro; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-03-02","source_content_hash_method":"raw_bytes","source_content_sha256":"c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0","source_published_at":"2026-03-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"67b82b3d54800e8378e5357b208e7d192247e30225c7df8f7d7fbaa13b033818","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.5,"score_display":"82.5","source_stated_rank":null},"run_fingerprint":"8eb7cf6080d4d5949b44a517ebc2102339590daeed7cf8a362deae972c072c4b","source":{"content_hash":"cd4cf4de3bf8c568c9c43f8211abbf4c12e41b290a91fce5f8fce5ee80023b44","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:25Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-9B","id":"qwen3-5-9b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; mmlu-pro; Qwen3.5-9B column","name":"Qwen3.5-9B official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-9B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b87cc44f5a3902b1b71e87ed","provider_config":{"source_id":"qwen3-5-9b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T14:31:08.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T14:31:08.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"9f2d391fd8c17da0b83d02e055510755022bb662d67cf341b5f52048ad408665","metric_details":{"best_score_across_scorers":"0.6166666666666667","model_release_date":"2026-02-24","stderr":6.003996985174109},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"YBMTDnW5LmCN6tBp2QYHzi","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYBMTDnW5LmCN6tBp2QYHzi.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/YBMTDnW5LmCN6tBp2QYHzi.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"05e2ba21e785d3047e3d2cac876f614eca4581d545cf95a07baa82d5f5802307","result":{"confidence_high":73.43450075760792,"confidence_low":49.89883257572542,"sample_count":45,"score":61.66666666666667,"score_display":"61.7","source_stated_rank":163},"run_fingerprint":"2cd097b9c078468d10f61598920e90350f1e3dc63541dbb885cdd293d0b538fb","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_4779204b36659935aea6f3dc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T14:30:39.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T14:30:39.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3a33c2713c9df5abb7786879dbcd1dc935419b5f2134d7f607ba7565d77aebad","metric_details":{"best_score_across_scorers":"0.42777777777777776","model_release_date":"2026-02-24","stderr":6.533819469304575},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"5FVmQUSS4uRd23f6nNgeRe","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5FVmQUSS4uRd23f6nNgeRe.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/5FVmQUSS4uRd23f6nNgeRe.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"ce493925119c725fddd9e7caeaab919fb4106ef26065d58cf4e916265f5a61af","result":{"confidence_high":55.584063937614744,"confidence_low":29.971491617940814,"sample_count":45,"score":42.77777777777778,"score_display":"42.8","source_stated_rank":203},"run_fingerprint":"02ddafcd40e458f7261cd991af872f24d241f2f16a725766daab30dc69bc7ab8","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0cec5caa07eafab37249d754","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"3bcbbaa9067c2d824e84c2d2958be87a196f85fa775e5cfaa69703585485e357","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-03-02","source_locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b","upstream_model_label":"Qwen3.5 9B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fe65d994034aeb7834d6b9554c9f405ccd73b0e1e75e414b5674a6358f556bbd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.248555,"score_display":"69.2","source_stated_rank":null},"run_fingerprint":"e4946e862847e6872eef63377c38da47cb3030e44c718a31542cace32ec00af1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7606f193c3b4b240833cf5022222ce7efbce78da978076f3458c727a1258cb9e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b-non-reasoning","upstream_model_label":"Qwen3.5 9B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2346686a5b4359ad036a9fcd19192882de7e91338f73b6f16450d95e752d8d91","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.763006,"score_display":"66.8","source_stated_rank":null},"run_fingerprint":"4c6b869f0b506056300b65054c168bc6c388588fb322110f57e6ecb67513cc31","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_4779204b36659935aea6f3dc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_028ab1c8abea206e4f16f3fd","split_or_window":"qwen-vendor-reported","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":217,"at":"2026-03-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T20:06:25Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-03-02","status":"stale"},"measurement_id":"2cee29af45aacdb3c4660fef3481087072fc3785079a256347fc2fc6bbac827f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/Qwen/Qwen3.5-9B/resolve/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md","evidence_sha256":"a1d07378ec333f3f6ff04e86aaaf4060e8cf3e1959123eb17fecb19e76dd0816","kind":"reviewed-static-report","published_at":"2026-03-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/Qwen/Qwen3.5-9B","version_id":"879c44ca5a6615044e90c0fad04eace605ab79c49db22a8f1684fafa29b2a0b4"},"source_locator":"Benchmark Results; MMMU-Pro row; Qwen3.5-9B column"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"MMMU-Pro; Qwen model-card evaluation. Exact task revision, sample count, and per-benchmark reasoning settings undisclosed; supplemental provider evidence, not a canonical organizer score.","reasoning_effort":"reported","result_published_at":"2026-03-02","source_content_hash_method":"raw_bytes","source_content_sha256":"c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0","source_published_at":"2026-03-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6be47b6349758cb0986680ba1e06f41d37d3190e39130dd45b80a09361d07ec7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.1,"score_display":"70.1","source_stated_rank":null},"run_fingerprint":"6bc20f416a057510587090c737c7de7b433c2f04cd5cda52580876d53182f290","source":{"content_hash":"cd4cf4de3bf8c568c9c43f8211abbf4c12e41b290a91fce5f8fce5ee80023b44","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:25Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-9B","id":"qwen3-5-9b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark Results; MMMU-Pro row; Qwen3.5-9B column","name":"Qwen3.5-9B official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-9B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b87cc44f5a3902b1b71e87ed","provider_config":{"source_id":"qwen3-5-9b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cf1180815fa72a1e22fa9ace461e836438cc0263903282e23d6810336afa8fe6","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-24","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.5 9B (Reasoning)","source_effort":null,"source_model_version":"qwen3.5-9B","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"eb5d8f43d44a026727508843f50d37f0465c3c4e7a27169902bd5f71453c441c","result":{"confidence_high":5.661392,"confidence_low":0.013679,"sample_count":71,"score":0.28571428571428603,"score_display":"0.3","source_stated_rank":156},"run_fingerprint":"6e49612bd5e2708497d31a41c54a96b4e48603d0db3d87bca559e48c1ec35adb","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2b7aa4882bff4522b1d7f304","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"097a5e333d4d30218f56381424e275a1599b9b5e2edb2f983ec68f964212d35a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-03-02","source_locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · critpt"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b","upstream_model_label":"Qwen3.5 9B (Reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"ddd3b01f5f32e139810ab49f371bc279ad64cbf56571377780a707ae9ec2af21","result":{"confidence_high":5.730683,"confidence_low":0.013504,"sample_count":70,"score":0.285714,"score_display":"0.3","source_stated_rank":null},"run_fingerprint":"10af4a39ac2623de8a89cee4c9fba5a97d030dd0d344fb7577f0f3d35082c529","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fe65e31468eb00b87c1763bb3d1548b5716ea6f3d6db04207c07d4b896da723f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · critpt"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b-non-reasoning","upstream_model_label":"Qwen3.5 9B (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"cab9b635f68754a3c3aa8127cd45603d8149a164973063f8dd66dfeb1b63a1f3","result":{"confidence_high":6.236252,"confidence_low":0.049636,"sample_count":70,"score":0.571429,"score_display":"0.6","source_stated_rank":null},"run_fingerprint":"56e4a5e8f94c629094f2523831ceaefc19b249de70a17e268de2b875bfd3273a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_4779204b36659935aea6f3dc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T14:31:04.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T14:31:04.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d0c7ec57494719e6ac8e45e9e258e10cb21c4acf9a6051bf31c88553b81f60b2","metric_details":{"best_score_across_scorers":"0.12","model_release_date":"2026-02-24","stderr":3.2659863237109037},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"i8CTho47u9qDcnXF3psREo","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fi8CTho47u9qDcnXF3psREo.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/i8CTho47u9qDcnXF3psREo.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"313511e3a0ce125da0c6567eebf55bc3dc41007b51fa0771cc6e356917fac3fc","result":{"confidence_high":18.40133319447337,"confidence_low":5.598666805526629,"sample_count":100,"score":12.0,"score_display":"12.0","source_stated_rank":127},"run_fingerprint":"71bb4fbbf86bee6feb6bf1426fbfa43711b98e9779b8814e160b83bc19b3ac90","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_4779204b36659935aea6f3dc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T14:30:31.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T14:30:31.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"38f43b6a44f55ca125b9b4ee16981bca634e1d0dae132e5a196700ab3e975edd","metric_details":{"best_score_across_scorers":"0.02","model_release_date":"2026-02-24","stderr":1.4070529413628967},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"4vjzjypxWVWBVdAnwaDQYX","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4vjzjypxWVWBVdAnwaDQYX.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/4vjzjypxWVWBVdAnwaDQYX.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"aa07e7db0e61e63d26a67a2863a819455cbec9b5c46b5e2c66cc053a668cd2db","result":{"confidence_high":4.757823765071278,"confidence_low":0.0,"sample_count":100,"score":2.0,"score_display":"2.0","source_stated_rank":185},"run_fingerprint":"ee342b5049aeee9d79acfd6041f7e32c01b15c07a2b4f120b8849a171face048","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1e48eb317a4e4da39effde5a","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_b530ca4da413885c54f52baa","split_or_window":"qwen-vendor-reported","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":217,"at":"2026-03-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-03-02","status":"stale"},"measurement_id":"ee0ed12422f2fea70a6d103fe70e39d4e43ab28813161dda0c31e04dcbf7a6dc","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/Qwen/Qwen3.5-9B/resolve/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md","evidence_sha256":"a1d07378ec333f3f6ff04e86aaaf4060e8cf3e1959123eb17fecb19e76dd0816","kind":"reviewed-static-report","published_at":"2026-03-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/Qwen/Qwen3.5-9B","version_id":"879c44ca5a6615044e90c0fad04eace605ab79c49db22a8f1684fafa29b2a0b4"},"source_locator":"Benchmark table; livecodebench; Qwen3.5-9B column"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"LiveCodeBench v6; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-03-02","source_content_hash_method":"raw_bytes","source_content_sha256":"c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0","source_published_at":"2026-03-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c1d1df0ddb34c3aad1342929c2b8d8d955114aae5c9a0d882037f2e4f2898440","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.6,"score_display":"65.6","source_stated_rank":null},"run_fingerprint":"a1d652cbfb6a18325c2ff5520422d392a72743d8246741fc8cdc51dfb05127fb","source":{"content_hash":"cd4cf4de3bf8c568c9c43f8211abbf4c12e41b290a91fce5f8fce5ee80023b44","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:25Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-9B","id":"qwen3-5-9b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; livecodebench; Qwen3.5-9B column","name":"Qwen3.5-9B official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-9B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b87cc44f5a3902b1b71e87ed","provider_config":{"source_id":"qwen3-5-9b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3d5dbae8eb5d1f2f5819c8b074d6374175cc3793eed3bc0d12a0bb061a83d191","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-24","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.5 9B (Reasoning)","source_effort":null,"source_model_version":"qwen3.5-9B","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"ce03d7b06d623fd647ca7fa0f501a311374ee36b58b6abddccbfb58c5e2868bb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":27.5462962962963,"score_display":"27.5","source_stated_rank":186},"run_fingerprint":"fb890c74a1eb2dad63fbc4162a0eba2b89a2eebe0043ee99056d79f7765f64cc","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b8beb3f11c16f95fe7bd0c8e","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"7c866d75eda84624e564e1dd09cfc05f38811d3082ac569c63bf4ebd6ea31ff2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-03-02","source_locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · scicode"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b","upstream_model_label":"Qwen3.5 9B (Reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"7a76e3cf48620097268ce4200901ee82d28bd05377d019d9406921cf37fec396","result":{"confidence_high":35.023458,"confidence_low":24.54365,"sample_count":288,"score":29.513889,"score_display":"29.5","source_stated_rank":null},"run_fingerprint":"8a3612a9f47aa67b334e69450db005a38200d90b59b2b3677bb12bd28e7cdae3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":217,"at":"2026-03-02","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-02","status":"stale"},"measurement_id":"d4f96238e484e933418c40c5d9180cfa7ff795459368956df2676b39200014cc","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-24","notes":null,"standard_error_points":2.3513565589,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Qwen3.5-9B","source_effort":null,"source_model_version":"qwen3.5-9B","source_row_date":"2026-03-02","source_scaffold":"little-coder","upstream_leaderboard_url":null},"run_count":null,"scaffold":"little-coder","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"fd5af75f494c11e03fc84e3f7920020c1f8b897753415dc1d4e75bac945bfe5a","result":{"confidence_high":13.822142001544,"confidence_low":4.6048242906559995,"sample_count":89,"score":9.2134831461,"score_display":"9.2","source_stated_rank":194},"run_fingerprint":"f2f611f2ab61362a326d09b9701b27aaf12114f5ea23bc70688445aff2d63539","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1a4157bb18d235747b3f390c","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"33e7bf3ccee9b93fbbc81029c5c2041c6f66e798c78520649229e092846486a5","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-03-02","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-5-9b","upstream_model_label":"Qwen3.5 9B (Reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"ba6768f1685c8cdcf9ce381fc6ee8390427b1989c374e28e7538dde56d725e04","result":{"confidence_high":39.363701,"confidence_low":20.783474,"sample_count":89,"score":29.213483,"score_display":"29.2","source_stated_rank":null},"run_fingerprint":"cc596828ec9189ece7e2331ac2e0150ff731e018226ca7139c045769eb7656b3","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a5edccf73e5c03c8f95abbd30b47a76ba131e5a71d56a3acb893413303f71ce1","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-03-02","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-5-9b-non-reasoning","upstream_model_label":"Qwen3.5 9B (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"e90654d5b6c96d9b469f4f36941e46de73ebdac70f4d3dbc00fa7e142ba2a04f","result":{"confidence_high":30.953042,"confidence_low":14.114687,"sample_count":89,"score":21.348315,"score_display":"21.3","source_stated_rank":null},"run_fingerprint":"2256851701d19e6ad64c9dc169d53be9945c8cf757e37859247b45cef2ed628b","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_4779204b36659935aea6f3dc","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"702a4c3430fe383d1abbc2d6ec9a743aecd40c21209d47cca17c542d7ed60862","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-03-02","source_locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · tauBanking"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-9b","upstream_model_label":"Qwen3.5 9B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"178df7deffbfba96e937f769542dc6a87b799bee15d0d78d2f9f4de7f6fea190","result":{"confidence_high":13.893584,"confidence_low":3.402452,"sample_count":97,"score":7.010309,"score_display":"7.0","source_stated_rank":null},"run_fingerprint":"5e3e6bbc14c8160d1bac39bb709d669aab348ca3c29c6e7f247b04fa98fdda31","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_367e04d56fad3a6f9849489d","split_or_window":"qwen-vendor-reported","unit":"percent","version":"BFCL-V4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":217,"at":"2026-03-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T20:06:25Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-03-02","status":"stale"},"measurement_id":"486d953a6edb19e2e0e1eac69de127d049a38ad4aa99be8d8dc5659d2a9f9dad","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/Qwen/Qwen3.5-9B/resolve/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md","evidence_sha256":"a1d07378ec333f3f6ff04e86aaaf4060e8cf3e1959123eb17fecb19e76dd0816","kind":"reviewed-static-report","published_at":"2026-03-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/Qwen/Qwen3.5-9B","version_id":"879c44ca5a6615044e90c0fad04eace605ab79c49db22a8f1684fafa29b2a0b4"},"source_locator":"Benchmark Results; BFCL-V4 row; Qwen3.5-9B column"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"BFCL-V4; Qwen model-card evaluation. Exact task revision, sample count, and per-benchmark reasoning settings undisclosed; supplemental provider evidence, not a canonical organizer score.","reasoning_effort":"reported","result_published_at":"2026-03-02","source_content_hash_method":"raw_bytes","source_content_sha256":"c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0","source_published_at":"2026-03-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"function-calling tools"},"run_count":null,"scaffold":null,"tools_allowed":"function-calling tools"},"protocol_fingerprint":"9d10e802f12470bbac18314ad12a3fbec3074eedd5d2186e2a365133438c65d4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":66.1,"score_display":"66.1","source_stated_rank":null},"run_fingerprint":"83a334dce84488221936c7a13fadd794bdb17317f106acc1dd477eae4afbd8b5","source":{"content_hash":"cd4cf4de3bf8c568c9c43f8211abbf4c12e41b290a91fce5f8fce5ee80023b44","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:25Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-9B","id":"qwen3-5-9b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark Results; BFCL-V4 row; Qwen3.5-9B column","name":"Qwen3.5-9B official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-9B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b87cc44f5a3902b1b71e87ed","provider_config":{"source_id":"qwen3-5-9b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifeval","metric":"accuracy","name":"IFEval","release_id":"release_eaa2f0262bb0264860cb2c8e","split_or_window":"qwen-vendor-reported","unit":"percent","version":"IFEval"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":217,"at":"2026-03-02","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:20Z","observed_at":"2026-10-04T14:29:20Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-03-02","status":"stale"},"measurement_id":"b843259777bba2aa4b266d44719f3606efdd96a235abb7aecd488390753637c6","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/Qwen/Qwen3.5-9B/resolve/c202236235762e1c871ad0ccb60c8ee5ba337b9a/README.md","evidence_sha256":"a1d07378ec333f3f6ff04e86aaaf4060e8cf3e1959123eb17fecb19e76dd0816","kind":"reviewed-static-report","published_at":"2026-03-02","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/Qwen/Qwen3.5-9B","version_id":"879c44ca5a6615044e90c0fad04eace605ab79c49db22a8f1684fafa29b2a0b4"},"source_locator":"Benchmark table; ifeval; Qwen3.5-9B column"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["ifeval_protocol_not_uniform"],"is_primary":false},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"IFEval; Qwen model-card protocol. Task revision, sample count and exact reasoning effort undisclosed; not equivalent to the organizer leaderboard.","reasoning_effort":"reported","result_published_at":"2026-03-02","source_content_hash_method":"raw_bytes","source_content_sha256":"c5f5a8c2dddab69cfbf05279235aa5fddb137939a06539c4c7637aa900fef6d0","source_published_at":"2026-03-02","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c2532bd711b5e3c0b76299a052a42563ef11942e70f026510be845076dcb1554","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":91.5,"score_display":"91.5","source_stated_rank":null},"run_fingerprint":"0d4d2a96a79a786100ab483068ec76c74fad3067e340e91b85af0a37a8facba3","source":{"content_hash":"cd4cf4de3bf8c568c9c43f8211abbf4c12e41b290a91fce5f8fce5ee80023b44","fetched_at":"2026-10-04T14:29:20Z","first_fetched_at":"2026-09-05T20:06:25Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-9B","id":"qwen3-5-9b-card","last_fetched_at":"2026-10-04T14:29:20Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; ifeval; Qwen3.5-9B column","name":"Qwen3.5-9B official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-9B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b87cc44f5a3902b1b71e87ed","provider_config":{"source_id":"qwen3-5-9b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a10dcaf4db45b0584ed6a6d92d3ca4b79fbea3b60d18fc9ad84bb3d632500513","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-03-02","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-5-9b","upstream_model_label":"Qwen3.5 9B (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"ca9d8d8277b398c1c9f053acd5e6a2a1d4aa207dde83eb75b2f7aca31c3e4fe1","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":413.53,"score_display":"414","source_stated_rank":null},"run_fingerprint":"74eab8e354d73d598f687f3d3ff70ac9e2f86b844a23621a9c6c5f9635efc475","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 9B (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifbench","metric":"inst_level_strict_acc","name":"IFBench","release_id":"release_4adf348ac2f76539ed7ccb10","split_or_window":"ifbench-test-300-prompts-344-instructions","unit":"percent","version":"swallow-evaluation-instruct"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-04T14:29:18Z","observed_at":"2026-10-04T14:29:18Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e22c80f41d0986c1a1b38bc1bbd57a06b2db83c2408e5c8d7e6f55dc74ae1b75","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","instruction_count":344,"model_release_date":"2026-03-02","prompt_count":300},"model":{"id":"alibaba/qwen3-5-9b","name":"Qwen 3.5 9B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"rule-based instruction checkers (lighteval 0.11.0 with Swallow fixes)","run_config":{"decoding":"temperature 0.6, top-p 0.95","leaderboard_revision":"c58a13a852eba061d639af7caf4ddbdff33b36e3","protocol":"Swallow LLM Leaderboard: swallow-evaluation-instruct (lighteval), allenai/IFBench_test, instruction-level strict accuracy; the final answer without the thinking part","swallow_model_id":"Qwen/Qwen3.5-9B","swallow_reasoning":"on"},"run_count":null,"scaffold":"swallow-evaluation-instruct (lighteval), swallow|ifbench_singleturn","tools_allowed":"none"},"protocol_fingerprint":"73ea367f7a86235a5d384e9dce2985cfcbe634f786e9dcc20c22225705467b05","result":{"confidence_high":null,"confidence_low":null,"sample_count":300,"score":48.3,"score_display":"48.3","source_stated_rank":null},"run_fingerprint":"d257907fc25eed9b044ee68378705daae998f98425c18d8b0f7f61d2fd2c23b6","source":{"content_hash":"fffdc5bb4b3de214a8a4ce885cf286e13a4b1488d9a4a0791e5918d1918b74cb","fetched_at":"2026-10-04T14:29:18Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://swallow-llm.github.io/leaderboard/index-post.en.html","id":"swallow-llm-leaderboard","last_fetched_at":"2026-10-04T14:29:18Z","license":"Leaderboard content CC-BY-4.0 (Swallow LLM Team, README of swallow-llm/leaderboard)","locator":null,"name":"Swallow LLM Leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/swallow-llm/leaderboard/c58a13a852eba061d639af7caf4ddbdff33b36e3/swallow_leaderboard_post_en.js"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_128d9317b7e94f2fdf3d00ed","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9c92c893051d1ce45b3d3139c2ac0eaf472b61fbf060466e3b2ff9c9e2da7595","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-04-29"},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen3-14B; model release: 2025-04-29","source_accessibility":"Open weights (unrestricted)","source_model_name":"Qwen3-14B","source_model_release_date":"2025-04-29","source_model_versions":["qwen3-14b","qwen3-14b_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"eac7aed1684306dd5023f6152da2ca7addb26f3a5c33e1b9df15411f8e7fb356","result":{"confidence_high":140.14,"confidence_low":133.48,"sample_count":null,"score":138.24,"score_display":"138.24","source_stated_rank":124},"run_fingerprint":"231ddd0c40c41b1dd001715b0beed173270527e49765a7289509798803336033","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_de4bc2746c30edb0d1633245","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T16:54:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T16:54:57.000Z","first_observed_at":"2026-09-01T00:20:17Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6036401dc25b34f72bf000d721be0818cec1ddbd9d9a3e2aec9f6686cd4f153a","metric_details":{"best_score_across_scorers":"0.5340909090909091","model_release_date":"2025-04-29","stderr":2.752462335115392},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"GRQV9Q8gHoHnSEPWofiQuq","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGRQV9Q8gHoHnSEPWofiQuq.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/GRQV9Q8gHoHnSEPWofiQuq.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"5b280138ae3a1b4b075a4a4d1589cbfc5ce70e41aaa4f0fa1b5d23f94de2d258","result":{"confidence_high":58.80391708591708,"confidence_low":48.014264732264735,"sample_count":null,"score":53.40909090909091,"score_display":"53.4","source_stated_rank":228},"run_fingerprint":"4d16ad6bc31bc22b83c0bcd55a4b7916a6166f829f99e564e0656971ed348bff","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_67bb468286cf106525571e45","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":37,"at":"2026-08-28T16:53:49.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T16:53:49.000Z","first_observed_at":"2026-09-01T00:20:17Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"23d28b3e971ed11223a0c5a179442d691b67a4f6a6c5788598a905750531f34a","metric_details":{"best_score_across_scorers":"0.6376262626262627","model_release_date":"2025-04-29","stderr":2.931953778086544},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"XGK9DjJHHVWr2LyT298ooL","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXGK9DjJHHVWr2LyT298ooL.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/XGK9DjJHHVWr2LyT298ooL.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"fda7c489e8f17a38157aad9268626a7698debb16ae0d63aabac728637d8a5ea3","result":{"confidence_high":69.50925566767589,"confidence_low":58.01599685757664,"sample_count":null,"score":63.76262626262626,"score_display":"63.8","source_stated_rank":204},"run_fingerprint":"2f2d10df0a967f0d4d366668e268280c7f0d6c87ee657629f1f65b291ea9d29f","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e536d85117176389c7a24936","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:42:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:42:57.000Z","first_observed_at":"2026-09-01T00:20:16Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d0efca6986b367bc401585285fc0404b2f64c4ea0022a67b6ac606e1b9b6a374","metric_details":{"best_score_across_scorers":"0.25833333333333336","model_release_date":"2025-04-29","stderr":4.823265376162594},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"RMcys99fv45vvgTgakXXWn","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRMcys99fv45vvgTgakXXWn.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/RMcys99fv45vvgTgakXXWn.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"82dd0f787fed28a4d341c9c175a630d392f770778c928a6945501cee25d76b47","result":{"confidence_high":35.28693347061202,"confidence_low":16.37973319605465,"sample_count":45,"score":25.833333333333336,"score_display":"25.8","source_stated_rank":227},"run_fingerprint":"a9471d9fc6dd26ee67608993497311139184cc7a280112d55fe86c74838a25b8","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_67bb468286cf106525571e45","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T16:54:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T16:54:14.000Z","first_observed_at":"2026-09-01T00:20:16Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1267c236a42b80ea34894a7eeac2dcac27488a638d785a2be149ad72e7475eb9","metric_details":{"best_score_across_scorers":"0.6638888888888889","model_release_date":"2025-04-29","stderr":6.1906554068112225},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gRRQHECkYhz5UaYsRKDh2F","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgRRQHECkYhz5UaYsRKDh2F.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/gRRQHECkYhz5UaYsRKDh2F.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5448de15e59f7d8f1d72845ae28cef2c748e36ac7e8fff32eb1cd12a5816f8a6","result":{"confidence_high":78.52257348623888,"confidence_low":54.25520429153889,"sample_count":45,"score":66.38888888888889,"score_display":"66.4","source_stated_rank":152},"run_fingerprint":"d60bcf311f637b37a6a0420ae81829b1d54d90a08fdab2acafe294f0e4a3b3db","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4a3b6cb0c6c555568a90dc7c","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:27:01Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d796372e182daa0f40fae142ffa825d7de826f7c32251395624e7973b6163a6b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-04-29","notes":null,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3 14B (Reasoning)","source_effort":null,"source_model_version":"qwen3-14b","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"9e57af0a6ede2d58f965f4e049058a4a2c8ad42b3a15fd865d21d3d2db170f48","result":{"confidence_high":5.132974,"confidence_low":0.0,"sample_count":71,"score":0.0,"score_display":"0.0","source_stated_rank":167},"run_fingerprint":"2d5e6af230df0d8a4892817057e5412edb0f076444676a88b416a8eee59d8e12","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_117d28b5512fedd5597760f4","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:42:55.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:42:55.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c81fef458199b6967062ee7e85d581af1f731d8dd8501b26bdd4fabc41928e44","metric_details":{"best_score_across_scorers":"0.0","model_release_date":"2025-04-29","stderr":0.0},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"oB8tcqTgWoZYX7BTFj4c8v","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoB8tcqTgWoZYX7BTFj4c8v.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/oB8tcqTgWoZYX7BTFj4c8v.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"bd4fefc3099dc230b2a5fa462986636378d18ed69706bc5f35ade12da9c2fff4","result":{"confidence_high":0.0,"confidence_low":0.0,"sample_count":100,"score":0.0,"score_display":"0.0","source_stated_rank":200},"run_fingerprint":"cbf5c287a26bed7599f0793931d26ae0deba7f08e319a8c28d5ab20839c24807","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_67bb468286cf106525571e45","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:42:37.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:42:37.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8c17147e69fb7c21928ff81a416eac2e8389b4771c688dce2ddc291cb298d603","metric_details":{"best_score_across_scorers":"0.04","model_release_date":"2025-04-29","stderr":1.9694638556693238},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"doo2bL7TEHLCNNMwf7NbYN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fdoo2bL7TEHLCNNMwf7NbYN.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/doo2bL7TEHLCNNMwf7NbYN.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5050e61f8e7f273b3f1cff315c5007e3b325fdbbbec510377e401b31ef26e11c","result":{"confidence_high":7.8601491571118745,"confidence_low":0.13985084288812555,"sample_count":100,"score":4.0,"score_display":"4.0","source_stated_rank":169},"run_fingerprint":"eef2b7c6f4b920ea579fdbadc22198210c5ec19f2ef7b01001721c04397f754e","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_773683f68f93ae6de41f71f0","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":32,"at":"2026-09-02T21:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T21:26:51Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2380614541ebffb66d557e8920d1caabf615fb717734f9bc186f19b734da8d00","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-04-29","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3 14B (Reasoning)","source_effort":null,"source_model_version":"qwen3-14b","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"14600173be83f44c993298618540e9f6ba4f2dbd4d19dbdd47f484352f9151ef","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.5972222222222,"score_display":"31.6","source_stated_rank":182},"run_fingerprint":"abc81b9b759f159d6e5044a7e772b1693f9c43e9b2c797333041ebbea22a6ba0","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f72a9c1ff904f7e8fb9475ac","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:03:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:03:18Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a4be7722c0edcbeca9ee305400a8f3df1d6a0b321ffa1f9ef51f7ad5c2eac59b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":4.5,"leaderboard_last_updated_at":"2026-04-12","license":"apache-2.0","model_link":"https://huggingface.co/Qwen/Qwen3-14B","total_cost_usd":3.38},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"FC"},"run_count":null,"scaffold":"FC","tools_allowed":null},"protocol_fingerprint":"6a9c99bccbca11748bdf70a2bdd6eb864726c94bf4a029f84ab95898fe96d192","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.03,"score_display":"41.03","source_stated_rank":43},"run_fingerprint":"3b39bae4fee1ecfc98e5010cad97c89a5c8d0375f97c6b10e4bf0e29760015dd","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b2ae49bd61766d7bcecc4aec","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":38,"at":"2026-08-27T22:30:42Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:42Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"07b3697445cb94795d540da7a6dd8adb34a437b6d2a8a99eeb1077ffa53e1d64","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":1.2,"leaderboard_last_updated_at":"2026-04-12","license":"apache-2.0","model_link":"https://huggingface.co/Qwen/Qwen3-14B","total_cost_usd":1.35},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prompted-function-calling","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"Prompt"},"run_count":null,"scaffold":"Prompt","tools_allowed":null},"protocol_fingerprint":"564fbea687964bbbeb4c2a4d92f977e2c00894a357d9ecf0f2b928b7737d37ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.77,"score_display":"37.77","source_stated_rank":47},"run_fingerprint":"7a1213742e62a07559be67bfd82fcb296ee32e599e4f2e186ef209138c4588be","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b2ae49bd61766d7bcecc4aec","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prompted-function-calling","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"appworld","metric":"task_goal_completion","name":"AppWorld","release_id":"release_3fa145932216c0afcdd9dae6","split_or_window":"test-challenge-all","unit":"percent","version":"official-c1f56015cf7c"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":232,"at":"2026-02-15","basis":"evaluation_at","evaluated_at":"2026-02-15","first_observed_at":"2026-09-02T21:28:04Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"add59be5c620d5f99ea80296255008411c5ed53e69b7eeb132c55657881ad2b5","metric_details":{"interactions":41.78,"normal_task_goal_completion":86.9,"scenario_goal_completion":50.4},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"leaderboard_row_id":"cabab065-0caee3c3","method":{"name":"Alibaba Cloud ApsaraLab AgentRL","tooltip":"Alibaba Cloud ApsaraLab AgentRL"},"repository_revision":"c1f56015cf7c3441ff1933f5bfbec879798d7bbe","source_model_label":{"name":"Qwen3-14B","tooltip":"Qwen3-14B"},"submission_url":"https://www.aliyun.com/product/bailian"},"run_count":null,"scaffold":"Alibaba Cloud ApsaraLab AgentRL","tools_allowed":"AppWorld APIs"},"protocol_fingerprint":"3fd2d05d1806fc66922d5d31d1ed44fa2ec28c31ae87e104d2ae6d50e87e1673","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":67.6,"score_display":"67.6","source_stated_rank":null},"run_fingerprint":"6a282aa75481b9e6924ce9566869eae06b580de362e97655cf399e9f83b2edb7","source":{"content_hash":"d44080bb01254150fab30f845d22262a28ee7c90f7850b98ce0ab235ea043f0f","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-09-02T21:28:04Z","homepage_url":"https://appworld.dev/leaderboard","id":"appworld","last_fetched_at":"2026-10-05T12:33:29Z","license":"Aggregate server-evaluated scores redistributable with attribution (2026-09-02)","locator":null,"name":"AppWorld official leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/StonyBrookNLP/appworld-leaderboard/c1f56015cf7c3441ff1933f5bfbec879798d7bbe/experiments/outputs/_leaderboard.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_65e947029c095ac93b048fea","provider_config":{"source_id":"appworld","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"23b9a886510e2c1560857ff78f18e6db441fa89a8f101dabaa8c2cf4d4371d09","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":32768,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2025-04-28","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Qwen3 14B (Reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-14b-instruct-reasoning","upstream_model_label":"Qwen3 14B (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"a9f9362f29a062a28f32b5cffcdc7acec788ae9e79d1c1041598d0734307f369","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":-38.78,"score_display":"-39","source_stated_rank":null},"run_fingerprint":"547ff2594aebffa42ac239248825bdb93e7391750febf55c35c72ca7a911d3a0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3 14B (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_c046a0e96997ecfa1f9b15d9","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifbench","metric":"inst_level_strict_acc","name":"IFBench","release_id":"release_4adf348ac2f76539ed7ccb10","split_or_window":"ifbench-test-300-prompts-344-instructions","unit":"percent","version":"swallow-evaluation-instruct"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-04T14:29:18Z","observed_at":"2026-10-04T14:29:18Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1c7cdf0ea5925ae7ddcbb9cb7dbbececbbcf8d408fa4522200c1eeb4f2aea355","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","instruction_count":344,"model_release_date":"2025-04-29","prompt_count":300},"model":{"id":"alibaba/qwen3-14b","name":"Qwen 3 14B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"rule-based instruction checkers (lighteval 0.11.0 with Swallow fixes)","run_config":{"decoding":"temperature 0.6, top-p 0.95","leaderboard_revision":"c58a13a852eba061d639af7caf4ddbdff33b36e3","protocol":"Swallow LLM Leaderboard: swallow-evaluation-instruct (lighteval), allenai/IFBench_test, instruction-level strict accuracy; the final answer without the thinking part","swallow_model_id":"Qwen/Qwen3-14B","swallow_reasoning":"on"},"run_count":null,"scaffold":"swallow-evaluation-instruct (lighteval), swallow|ifbench_singleturn","tools_allowed":"none"},"protocol_fingerprint":"6492833b49c17c6db29dd2299cdcda765ec1357b6c689b61d0600cba0e168682","result":{"confidence_high":null,"confidence_low":null,"sample_count":300,"score":32.300000000000004,"score_display":"32.3","source_stated_rank":null},"run_fingerprint":"b34adedafb1fde9d3f452badf0b307a7770fc4c905dcecfaa026dcd714a115b0","source":{"content_hash":"fffdc5bb4b3de214a8a4ce885cf286e13a4b1488d9a4a0791e5918d1918b74cb","fetched_at":"2026-10-04T14:29:18Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://swallow-llm.github.io/leaderboard/index-post.en.html","id":"swallow-llm-leaderboard","last_fetched_at":"2026-10-04T14:29:18Z","license":"Leaderboard content CC-BY-4.0 (Swallow LLM Team, README of swallow-llm/leaderboard)","locator":null,"name":"Swallow LLM Leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/swallow-llm/leaderboard/c58a13a852eba061d639af7caf4ddbdff33b36e3/swallow_leaderboard_post_en.js"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_c046a0e96997ecfa1f9b15d9","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_9b8cd6ba0a835eb2da22bb93","split_or_window":"google-vendor-reported","unit":"percent","version":"HLE"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T20:06:27Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"d18c8a8cd0cca8206b37c89c6780e6765d3e737fdeb12375abd7ee6aa3847428","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-12B-it/resolve/707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7/README.md","evidence_sha256":"0537da7712413a1289e98876a3c6e8991597edcfcf26083a16f1609b51f87fb8","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-12B-it","version_id":"24cd65640c7bffcd9e480146106d9e23e94eeadabc78655283a1892a94526297"},"source_locator":"Benchmark Results; HLE no tools row; Gemma 4 12B IT column"},"model":{"id":"google/gemma-4-12b-it","name":"Gemma 4 12B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"HLE no tools; Google model-card evaluation. Exact task revision, sample count, and per-benchmark reasoning settings undisclosed; supplemental provider evidence, not a canonical organizer score.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"04a06469e3d83b941eaa776a3647b8690b5a7cc5c833af2381e5345bb1bb45c0","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c7b0e4330c983c8b61ff308934dd9e46ac24e7b939d101095153586aa5e21a2e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5.2,"score_display":"5.2","source_stated_rank":null},"run_fingerprint":"65a5492c7a96153ccf605341e32e1777d82d4e3af9fe4ab26af91fb02872f5af","source":{"content_hash":"8eb4131406106ae8a65d0fd815fbbbe456c83d1024b7ce6204006cccbc09a193","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:27Z","homepage_url":"https://huggingface.co/google/gemma-4-12B-it","id":"gemma4-12b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark Results; HLE no tools row; Gemma 4 12B IT column","name":"Gemma 4 12B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-12B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3b514549614b819522b86adb","provider_config":{"source_id":"gemma4-12b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_fa97cbc1bde3163225d15751","split_or_window":"google-vendor-reported","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"67c3d42930ed74602bce0f3e6aeb82a997165c83a9f680222b27e4071b4da4c8","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-12B-it/resolve/707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7/README.md","evidence_sha256":"0537da7712413a1289e98876a3c6e8991597edcfcf26083a16f1609b51f87fb8","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-12B-it","version_id":"24cd65640c7bffcd9e480146106d9e23e94eeadabc78655283a1892a94526297"},"source_locator":"Benchmark table; gpqa-diamond; Gemma 4 12B IT column"},"model":{"id":"google/gemma-4-12b-it","name":"Gemma 4 12B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"GPQA Diamond; official Google model-card run with thinking. Exact task snapshot, repetitions and inference budget undisclosed.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"04a06469e3d83b941eaa776a3647b8690b5a7cc5c833af2381e5345bb1bb45c0","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"8bb642c88acb255828e2b9b1f8de45d7378e2b4db0e5af2b1f5cc9c54331afa8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.8,"score_display":"78.8","source_stated_rank":null},"run_fingerprint":"8263e40bd6175aec016ada4a40dd3693cdc0838024773b66d3360b42b260f37b","source":{"content_hash":"8eb4131406106ae8a65d0fd815fbbbe456c83d1024b7ce6204006cccbc09a193","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:27Z","homepage_url":"https://huggingface.co/google/gemma-4-12B-it","id":"gemma4-12b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; gpqa-diamond; Gemma 4 12B IT column","name":"Gemma 4 12B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-12B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3b514549614b819522b86adb","provider_config":{"source_id":"gemma4-12b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_cbef64e634e73d9217bfe565","split_or_window":"google-vendor-reported","unit":"percent","version":"MMLU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"77500c915baa81e8fac0378c8847e20f2312c0a92e6be77494f87723f84d56b3","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-12B-it/resolve/707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7/README.md","evidence_sha256":"0537da7712413a1289e98876a3c6e8991597edcfcf26083a16f1609b51f87fb8","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-12B-it","version_id":"24cd65640c7bffcd9e480146106d9e23e94eeadabc78655283a1892a94526297"},"source_locator":"Benchmark table; mmlu-pro; Gemma 4 12B IT column"},"model":{"id":"google/gemma-4-12b-it","name":"Gemma 4 12B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"MMLU Pro; official Google model-card run with thinking. Exact task snapshot, repetitions and inference budget undisclosed.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"04a06469e3d83b941eaa776a3647b8690b5a7cc5c833af2381e5345bb1bb45c0","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"701b19bfac3ec0298ae4580dfb7a13feef9ff9e77cd0cb2cdc7e0a0e8c25976c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.2,"score_display":"77.2","source_stated_rank":null},"run_fingerprint":"15d413232dacf9e401f359fb25b4666e595a7d36996e8ef531621ecb52c1cd3e","source":{"content_hash":"8eb4131406106ae8a65d0fd815fbbbe456c83d1024b7ce6204006cccbc09a193","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:27Z","homepage_url":"https://huggingface.co/google/gemma-4-12B-it","id":"gemma4-12b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; mmlu-pro; Gemma 4 12B IT column","name":"Gemma 4 12B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-12B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3b514549614b819522b86adb","provider_config":{"source_id":"gemma4-12b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_6f3fd0029277bfcad78a10b3","split_or_window":"google-vendor-reported","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T20:06:27Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"8c846ddbb6c8348543ac851b4b044db6ddbe2f58983bc494b688f2d62e321aad","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-12B-it/resolve/707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7/README.md","evidence_sha256":"0537da7712413a1289e98876a3c6e8991597edcfcf26083a16f1609b51f87fb8","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-12B-it","version_id":"24cd65640c7bffcd9e480146106d9e23e94eeadabc78655283a1892a94526297"},"source_locator":"Benchmark Results; MMMU Pro row; Gemma 4 12B IT column"},"model":{"id":"google/gemma-4-12b-it","name":"Gemma 4 12B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"MMMU Pro; Google model-card evaluation. Exact task revision, sample count, and per-benchmark reasoning settings undisclosed; supplemental provider evidence, not a canonical organizer score.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"04a06469e3d83b941eaa776a3647b8690b5a7cc5c833af2381e5345bb1bb45c0","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"bb9f70a6a5495741815c8a649fc71037491b19c339ce5782e0a65c139014c769","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":69.1,"score_display":"69.1","source_stated_rank":null},"run_fingerprint":"8c4561873e44e627448e353e0e22d3e9652c4cdbd34908950291d7878e414f90","source":{"content_hash":"8eb4131406106ae8a65d0fd815fbbbe456c83d1024b7ce6204006cccbc09a193","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:27Z","homepage_url":"https://huggingface.co/google/gemma-4-12B-it","id":"gemma4-12b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark Results; MMMU Pro row; Gemma 4 12B IT column","name":"Gemma 4 12B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-12B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3b514549614b819522b86adb","provider_config":{"source_id":"gemma4-12b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_1d1cf16bb310d2fb0576bb02","split_or_window":"google-vendor-reported","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"cc47b23796a94a95e343fefcfcb69a0d5b2aa87ffdc24c765e18a6fc3456447d","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-12B-it/resolve/707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7/README.md","evidence_sha256":"0537da7712413a1289e98876a3c6e8991597edcfcf26083a16f1609b51f87fb8","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-12B-it","version_id":"24cd65640c7bffcd9e480146106d9e23e94eeadabc78655283a1892a94526297"},"source_locator":"Benchmark table; livecodebench; Gemma 4 12B IT column"},"model":{"id":"google/gemma-4-12b-it","name":"Gemma 4 12B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"LiveCodeBench v6; official Google model-card run with thinking. Exact task snapshot, repetitions and inference budget undisclosed.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"04a06469e3d83b941eaa776a3647b8690b5a7cc5c833af2381e5345bb1bb45c0","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"0b730d9b32134e49951ef8ed8d71eb2e3dae7bc469983e225238f4cef7d267d9","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.0,"score_display":"72.0","source_stated_rank":null},"run_fingerprint":"56e72150ed4cc6072c9711350da5acb66bfe7da1b075cd89d887fa7da483cb33","source":{"content_hash":"8eb4131406106ae8a65d0fd815fbbbe456c83d1024b7ce6204006cccbc09a193","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:27Z","homepage_url":"https://huggingface.co/google/gemma-4-12B-it","id":"gemma4-12b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; livecodebench; Gemma 4 12B IT column","name":"Gemma 4 12B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-12B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3b514549614b819522b86adb","provider_config":{"source_id":"gemma4-12b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_9b8cd6ba0a835eb2da22bb93","split_or_window":"google-vendor-reported","unit":"percent","version":"HLE"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T20:06:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"4cfc9930b1d7c404ed6d5762b3f3af2f5228e546e778ace5a674dcbd593bff87","metric_details":{"comparison_warning":"HLE full multimodal (2,500 tasks), without tools. The report date is only a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"hle-no-tools-report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-26B-A4B-it/resolve/4d7ae4984b7db7de8f8457170b3f1a419ee76d52/README.md","evidence_sha256":"5502b7bef29d2ee787ca365a4ef8e55e2151bf8af2d5835f00c325966720e149","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","version_id":"b1a43ec0ffd68e5e847abce63a5084680a529e31d588675518ad887d97d50b7f"},"source_locator":"Benchmark Results; HLE no tools row; Gemma 4 26B A4B IT column"},"model":{"id":"google/gemma-4-26b-a4b-it","name":"Gemma 4 26B-A4B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"hle-no-tools-report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"HLE no tools; Google model-card evaluation. Exact task revision, sample count, and per-benchmark reasoning settings undisclosed; supplemental provider evidence, not a canonical organizer score.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"4f463225c4fe49ec120115b0f9bb6713ff1556293a909903ffdae3df7bf974da","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"06ac0b5ca9aa6c39ffe98cb437a39fcfcce2acb7262c96bbdc1e654d9cc4bbc0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":8.7,"score_display":"8.7","source_stated_rank":null},"run_fingerprint":"02d5ef20549391d85a4ec163bbee89739ec5c80cf2bc5017739c3f0e1bee36a9","source":{"content_hash":"7394c7733f1a79ca06eba9e30de2fd40f73e63a666dd2e2b9fa0e65e6a9d2ee5","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","id":"gemma4-26b-a4b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark Results; HLE no tools row; Gemma 4 26B A4B IT column","name":"Gemma 4 26B A4B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-26B-A4B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6f7c719dfe08806351b5e839","provider_config":{"source_id":"gemma4-26b-a4b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T23:59:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T23:59:25.000Z","first_observed_at":"2026-08-28T00:52:16Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"12dbdf1d9ebd0bd1b44c020506e1c49041359129e37472e880d053f74dad920f","metric_details":{"best_score_across_scorers":"0.7323232323232324","model_release_date":"2026-04-02","stderr":3.1544498882702823},"model":{"id":"google/gemma-4-26b-a4b-it","name":"Gemma 4 26B-A4B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Usx9KaiXvHZU4qrzgc4sLg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUsx9KaiXvHZU4qrzgc4sLg.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Usx9KaiXvHZU4qrzgc4sLg.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"a1c40e0218605478104c85e1c23125c3f8ab74131ce10b070928d9226d484a23","result":{"confidence_high":79.41504501333299,"confidence_low":67.04960145131349,"sample_count":null,"score":73.23232323232324,"score_display":"73.2","source_stated_rank":167},"run_fingerprint":"b0164b3d9d6a8800bb3b166eecaa35169778ade3c286126a8e5f7646e8e871ed","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_bc8fb7aacaacf9a5181ca317","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_fa97cbc1bde3163225d15751","split_or_window":"google-vendor-reported","unit":"percent","version":"GPQA Diamond"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"46ec4dc5bf7d9bd96489b179d7c2b9177cd8c074b76aa3f5d3ec97f1f81e8c22","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-26B-A4B-it/resolve/4d7ae4984b7db7de8f8457170b3f1a419ee76d52/README.md","evidence_sha256":"5502b7bef29d2ee787ca365a4ef8e55e2151bf8af2d5835f00c325966720e149","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","version_id":"b1a43ec0ffd68e5e847abce63a5084680a529e31d588675518ad887d97d50b7f"},"source_locator":"Benchmark table; gpqa-diamond; Gemma 4 26B A4B IT column"},"model":{"id":"google/gemma-4-26b-a4b-it","name":"Gemma 4 26B-A4B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"GPQA Diamond; official Google model-card run with thinking. Exact task snapshot, repetitions and inference budget undisclosed.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"4f463225c4fe49ec120115b0f9bb6713ff1556293a909903ffdae3df7bf974da","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a6d022205a3aeada29f3d68a36e3fe42e0af4789ee322a5bb29e9ca210f1027e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.3,"score_display":"82.3","source_stated_rank":null},"run_fingerprint":"5906196d007369ec00446a85e6d22f1df04518a31a6337c9dc8aff14c28c7631","source":{"content_hash":"7394c7733f1a79ca06eba9e30de2fd40f73e63a666dd2e2b9fa0e65e6a9d2ee5","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","id":"gemma4-26b-a4b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; gpqa-diamond; Gemma 4 26B A4B IT column","name":"Gemma 4 26B A4B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-26B-A4B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6f7c719dfe08806351b5e839","provider_config":{"source_id":"gemma4-26b-a4b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_cbef64e634e73d9217bfe565","split_or_window":"google-vendor-reported","unit":"percent","version":"MMLU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"7957c91bbd25f5eef424f8f3afce09cc57a5062ba5d640e1681cb4f3579a8360","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-26B-A4B-it/resolve/4d7ae4984b7db7de8f8457170b3f1a419ee76d52/README.md","evidence_sha256":"5502b7bef29d2ee787ca365a4ef8e55e2151bf8af2d5835f00c325966720e149","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","version_id":"b1a43ec0ffd68e5e847abce63a5084680a529e31d588675518ad887d97d50b7f"},"source_locator":"Benchmark table; mmlu-pro; Gemma 4 26B A4B IT column"},"model":{"id":"google/gemma-4-26b-a4b-it","name":"Gemma 4 26B-A4B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"MMLU Pro; official Google model-card run with thinking. Exact task snapshot, repetitions and inference budget undisclosed.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"4f463225c4fe49ec120115b0f9bb6713ff1556293a909903ffdae3df7bf974da","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a2a0026bdf12f6d0b107ae36753b679cab04abadefa418ed0920e49d04843f0f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":82.6,"score_display":"82.6","source_stated_rank":null},"run_fingerprint":"84b4cfcca39babd5b69aa7a9cb6ac3b160709d817affe063345ec7428ffe3a58","source":{"content_hash":"7394c7733f1a79ca06eba9e30de2fd40f73e63a666dd2e2b9fa0e65e6a9d2ee5","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","id":"gemma4-26b-a4b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; mmlu-pro; Gemma 4 26B A4B IT column","name":"Gemma 4 26B A4B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-26B-A4B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6f7c719dfe08806351b5e839","provider_config":{"source_id":"gemma4-26b-a4b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-28T12:55:56.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T12:55:56.000Z","first_observed_at":"2026-08-29T21:50:50Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0730fa29208c8d8a8a375273797a1500b31dc6148decac0c43fd69a23d03b592","metric_details":{"best_score_across_scorers":"0.8222222222222222","model_release_date":"2026-04-02","stderr":5.763774795025093},"model":{"id":"google/gemma-4-26b-a4b-it","name":"Gemma 4 26B-A4B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mstwSH7noJfvhGcd6Lz5kd","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmstwSH7noJfvhGcd6Lz5kd.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mstwSH7noJfvhGcd6Lz5kd.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"e34b79d4bd0fd4a67e6ab3ac50f1df288ff091726118fba383eb00ed5ec197be","result":{"confidence_high":93.51922082047139,"confidence_low":70.92522362397304,"sample_count":45,"score":82.22222222222221,"score_display":"82.2","source_stated_rank":103},"run_fingerprint":"4f949b24d4b787557730b8b78027e1a57e7e22132681ed285d8a4d48fa46561f","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_bc8fb7aacaacf9a5181ca317","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_6f3fd0029277bfcad78a10b3","split_or_window":"google-vendor-reported","unit":"percent","version":"MMMU-Pro"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T20:06:26Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"689a8540de8f3c7a5f0674146ba0f31668a064c5903ecaca16d601bc0d6cf1ed","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-26B-A4B-it/resolve/4d7ae4984b7db7de8f8457170b3f1a419ee76d52/README.md","evidence_sha256":"5502b7bef29d2ee787ca365a4ef8e55e2151bf8af2d5835f00c325966720e149","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","version_id":"b1a43ec0ffd68e5e847abce63a5084680a529e31d588675518ad887d97d50b7f"},"source_locator":"Benchmark Results; MMMU Pro row; Gemma 4 26B A4B IT column"},"model":{"id":"google/gemma-4-26b-a4b-it","name":"Gemma 4 26B-A4B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"MMMU Pro; Google model-card evaluation. Exact task revision, sample count, and per-benchmark reasoning settings undisclosed; supplemental provider evidence, not a canonical organizer score.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"4f463225c4fe49ec120115b0f9bb6713ff1556293a909903ffdae3df7bf974da","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6d01ec25a9572f1b2135d4ad993b46c30e87e32cbb3c89a35cf3c019bba2bd3d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.8,"score_display":"73.8","source_stated_rank":null},"run_fingerprint":"149ac26e99ff80e5f298c6f184758c11b986fc881eb925310bf9df892ee98226","source":{"content_hash":"7394c7733f1a79ca06eba9e30de2fd40f73e63a666dd2e2b9fa0e65e6a9d2ee5","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","id":"gemma4-26b-a4b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark Results; MMMU Pro row; Gemma 4 26B A4B IT column","name":"Gemma 4 26B A4B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-26B-A4B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6f7c719dfe08806351b5e839","provider_config":{"source_id":"gemma4-26b-a4b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:36:15.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:36:15.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3d7011f08decd029bbe3bb9b8ff10a07eebfa6ad3d339db3cb2823edc9c32cad","metric_details":{"best_score_across_scorers":"0.06","model_release_date":"2026-04-02","stderr":2.3868325657594203},"model":{"id":"google/gemma-4-26b-a4b-it","name":"Gemma 4 26B-A4B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"272CatAfNpHUw44w7oJ2MM","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F272CatAfNpHUw44w7oJ2MM.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/272CatAfNpHUw44w7oJ2MM.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9ffd08dd725903834e09b87baa52b74e8d4284c90c73e0358f3d461e00d2d9bd","result":{"confidence_high":10.678191828888464,"confidence_low":1.3218081711115364,"sample_count":100,"score":6.0,"score_display":"6.0","source_stated_rank":155},"run_fingerprint":"b68695a0fe1960afb0f2aba49a72c0b3e27946b2aaebf17985985da67b5cbb8d","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_bc8fb7aacaacf9a5181ca317","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass@1","name":"LiveCodeBench","release_id":"release_1d1cf16bb310d2fb0576bb02","split_or_window":"google-vendor-reported","unit":"percent","version":"v6"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":77,"at":"2026-07-20","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-05T16:42:25Z","last_confirmed_at":"2026-10-04T14:29:21Z","observed_at":"2026-10-04T14:29:21Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-07-20","status":"fresh"},"measurement_id":"a45a3ce9fb128956b56f5cf7a857ed169bd7c6bb501a49150c87e22eb924dbbc","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_document_version":{"document_url":"https://huggingface.co/google/gemma-4-26B-A4B-it/resolve/4d7ae4984b7db7de8f8457170b3f1a419ee76d52/README.md","evidence_sha256":"5502b7bef29d2ee787ca365a4ef8e55e2151bf8af2d5835f00c325966720e149","kind":"reviewed-static-report","published_at":"2026-07-20","reviewed_at":"2026-09-05","source_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","version_id":"b1a43ec0ffd68e5e847abce63a5084680a529e31d588675518ad887d97d50b7f"},"source_locator":"Benchmark table; livecodebench; Gemma 4 26B A4B IT column"},"model":{"id":"google/gemma-4-26b-a4b-it","name":"Gemma 4 26B-A4B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Google","evidence_verified":true,"freshness_proxy":true,"protocol":"LiveCodeBench v6; official Google model-card run with thinking. Exact task snapshot, repetitions and inference budget undisclosed.","reasoning_effort":"reported","result_published_at":"2026-07-20","source_content_hash_method":"raw_bytes","source_content_sha256":"4f463225c4fe49ec120115b0f9bb6713ff1556293a909903ffdae3df7bf974da","source_published_at":"2026-07-20","source_published_at_evidence":"Publication date of the reviewed source revision; original execution dates are not independently verified.","source_published_at_precision":"date","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5d31469ed4451daa67379e297cddd9f64424f87add212e1bf0b12450d9b3b79f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.1,"score_display":"77.1","source_stated_rank":null},"run_fingerprint":"13120c5ba16380749a067f7ac080a75d046a91306a339c88986d5c0d14a61c51","source":{"content_hash":"7394c7733f1a79ca06eba9e30de2fd40f73e63a666dd2e2b9fa0e65e6a9d2ee5","fetched_at":"2026-10-04T14:29:21Z","first_fetched_at":"2026-09-05T20:06:26Z","homepage_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","id":"gemma4-26b-a4b-card","last_fetched_at":"2026-10-04T14:29:21Z","license":"Aggregate score facts with attribution; upstream document terms apply","locator":"Benchmark table; livecodebench; Gemma 4 26B A4B IT column","name":"Gemma 4 26B A4B IT official model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/google/gemma-4-26B-A4B-it"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6f7c719dfe08806351b5e839","provider_config":{"source_id":"gemma4-26b-a4b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"instruction-following","higher_is_better":true,"id":"ifbench","metric":"inst_level_strict_acc","name":"IFBench","release_id":"release_4adf348ac2f76539ed7ccb10","split_or_window":"ifbench-test-300-prompts-344-instructions","unit":"percent","version":"swallow-evaluation-instruct"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:19Z","last_confirmed_at":"2026-10-04T14:29:18Z","observed_at":"2026-10-04T14:29:18Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bdcef8a4037ad26843447cdb8142c10cfd48493a506b85d5647dd89769baf493","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","instruction_count":344,"model_release_date":"2026-04-02","prompt_count":300},"model":{"id":"google/gemma-4-26b-a4b-it","name":"Gemma 4 26B-A4B IT","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"rule-based instruction checkers (lighteval 0.11.0 with Swallow fixes)","run_config":{"decoding":"temperature 0.6, top-p 0.95","leaderboard_revision":"c58a13a852eba061d639af7caf4ddbdff33b36e3","protocol":"Swallow LLM Leaderboard: swallow-evaluation-instruct (lighteval), allenai/IFBench_test, instruction-level strict accuracy; the final answer without the thinking part","swallow_model_id":"google/gemma-4-26B-A4B-it","swallow_reasoning":"on"},"run_count":null,"scaffold":"swallow-evaluation-instruct (lighteval), swallow|ifbench_singleturn","tools_allowed":"none"},"protocol_fingerprint":"f55d1bb34adc6d2f246c79d0307e2ab6ee1679c31c3304e9628c0cdc7e169f71","result":{"confidence_high":null,"confidence_low":null,"sample_count":300,"score":72.7,"score_display":"72.7","source_stated_rank":null},"run_fingerprint":"5ebf3111b8bd70382c9189a6fc80a52c24cad87741579f76b3f7437c100b588c","source":{"content_hash":"fffdc5bb4b3de214a8a4ce885cf286e13a4b1488d9a4a0791e5918d1918b74cb","fetched_at":"2026-10-04T14:29:18Z","first_fetched_at":"2026-09-30T18:53:19Z","homepage_url":"https://swallow-llm.github.io/leaderboard/index-post.en.html","id":"swallow-llm-leaderboard","last_fetched_at":"2026-10-04T14:29:18Z","license":"Leaderboard content CC-BY-4.0 (Swallow LLM Team, README of swallow-llm/leaderboard)","locator":null,"name":"Swallow LLM Leaderboard","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/swallow-llm/leaderboard/c58a13a852eba061d639af7caf4ddbdff33b36e3/swallow_leaderboard_post_en.js"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_15d0ef6dad8ff8c425a50e57","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"27aa8f25dddf039b9e545c30554319a584ca970b66db5cf451d9744183c54f72","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-01-27"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Kimi K2.5; model release: 2026-01-27","source_accessibility":"Open weights (unrestricted)","source_model_name":"Kimi K2.5","source_model_release_date":"2026-01-27","source_model_versions":["fireworks/kimi-k2p5","kimi-k2.5"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6231b6ebf0d5590f6b3f54deea320fd42b1a23446e2223ecc7854028e1806e43","result":{"confidence_high":149.32,"confidence_low":146.45,"sample_count":null,"score":148.04,"score_display":"148.04","source_stated_rank":60},"run_fingerprint":"06e84b0e3daecd09ec3efa5c7c46263df6718104ef781747ff93a06d4490f683","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_319ef04e6ebcddf6677b39cd","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"db35d3028edccd834395f2b4d9ae291f561018613e85ff61facd455de365ec9a","metric_details":{"license":"Modified MIT","variance":2.813405329751386},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1453.4093382997776,"confidence_low":1446.8343612746735,"sample_count":74565,"score":1450.1218497872255,"score_display":"1450","source_stated_rank":78},"run_fingerprint":"5aa2ddec3b3d3cebd0a7083a0ad14babc6dc67b0cf3a2be7f0f547377b097bdb","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":233,"at":"2026-02-13T14:32:29Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-13T14:32:29Z","status":"stale"},"measurement_id":"a18fdaf9b32ced965c236795f8e7871483d5d70d146878c08acd10d61a71dcc1","metric_details":{"calibration_error":67.0,"confidence_interval_half_width":1.81,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=kimi-k2.5; createdAt=2026-02-13T14:32:29Z","source_row_created_at":"2026-02-13T14:32:29Z"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"kimi-k2.5","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9582fb3c52292a203b63b834c090dff052133ee6596e5a6d0f0902b3bbef79f1","result":{"confidence_high":26.18,"confidence_low":22.560000000000002,"sample_count":2500,"score":24.37,"score_display":"24.37","source_stated_rank":10},"run_fingerprint":"c63a817c9c687fd0d76c15176d7aac67a2e0b54680c6f31d2f19712642f5f507","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=kimi-k2.5; createdAt=2026-02-13T14:32:29Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a22fc3d76219aa5ad672fcbb","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"800df649c19517587e641fe6b4ba7e62e0c58800ba1a14228fbeba271f1073d3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":null,"num_parameters":1026879376368,"pull_request":126,"result_file_url":"https://huggingface.co/moonshotai/Kimi-K2.5/resolve/refs%2Fpr%2F126/.eval_results/hle.yaml","source":{"isExternal":true,"name":"EvalEval","url":"https://huggingface.co/datasets/evaleval/EEE_datastore/blob/192329fb7d6b15b7b0936a1a58ae862aa7e8ba24/flat/objects/aa/05/aa055fe5-5174-4517-8b10-19d46da94f3e.json"}},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.37,"score_display":"24.37","source_stated_rank":52},"run_fingerprint":"3d1b0e4fb653983f1eced569fcdd478326a42ab05355143bebcc4ee2b31bfddf","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a5878b02d4896219e8e715ce","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":36,"at":"2026-08-29T23:52:43Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"28d5cd01f429e1bf09522f391e44833d7602a1e6f636373a0e077ce4b9ab42bb","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","modality_lane":"not reported","notes":"With tools","num_parameters":1026879376368,"pull_request":49,"result_file_url":"https://huggingface.co/moonshotai/Kimi-K2.5/resolve/refs%2Fpr%2F49/.eval_results/hle_with_tools.yaml","source":{"author":{"_id":"5f1158120c833276f61f1a84","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1608042047613-5f1158120c833276f61f1a84.jpeg","followerCount":1327,"fullname":"Niels Rogge","isHf":true,"isHfAdmin":false,"isMod":false,"isPro":false,"isUserFollowing":false,"name":"nielsr","type":"user"},"isExternal":false,"name":"Model Card","url":"https://huggingface.co/moonshotai/Kimi-K2.5"}},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.2,"score_display":"50.2","source_stated_rank":14},"run_fingerprint":"a4d08c5f6052e24b8334a6a73fdc23d332b9c4d6d5130254f4f98938436bd4b5","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a5878b02d4896219e8e715ce","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T19:35:45.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-27T19:35:45.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3f95baa8146cfe78709f1a876a86e8693febb2146e67fb2e71958cd0ed576c0e","metric_details":{"best_score_across_scorers":"0.343","model_release_date":"2026-01-27","stderr":1.5019206922357007},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"RbXXg4JEbTSBnzw3CMSHnD","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRbXXg4JEbTSBnzw3CMSHnD.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/RbXXg4JEbTSBnzw3CMSHnD.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"9b27a207e2a514be653bf3478a54786b2e28f6a37c4d6332e9e947a6618acd03","result":{"confidence_high":37.243764556781976,"confidence_low":31.356235443218033,"sample_count":1000,"score":34.300000000000004,"score_display":"34.3","source_stated_rank":52},"run_fingerprint":"2e025cd3cd7ec492893fe28cfdb03b80dc9c0bdabf8263418356612d8bb16a24","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7aaddd2f0cc6c1a8ff26df64","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"29e66253a35c9fa55af447490a03cffe2fa65dca0fc079e6165833b9622bf591","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.029962,"date_is_proxy":true,"latency_seconds":151.843,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.135},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":82.828,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":128000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.5-thinking","zero_shot_accuracy":85.354},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"b3e2b4c042655bf008527842f38ce8754491cc4b3e95174642a7508afdf14570","result":{"confidence_high":88.2756,"confidence_low":79.90639999999999,"sample_count":396,"score":84.091,"score_display":"84.09","source_stated_rank":54},"run_fingerprint":"934c0f75e1a1b08666fb65d5e3a21d9a48a19409920faf6f9c97aee8a31d84ba","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":245,"at":"2026-02-02T11:01:01.260Z","basis":"evaluation_started_at","evaluated_at":"2026-02-02T11:01:01.260Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"958352f0fe7d3dd51e4e9bed828d352a641b9327d876ad808ac34b7a8a010ce7","metric_details":{"best_score_across_scorers":"0.876","model_release_date":"2026-01-27","stderr":1.9},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"FmqGUq87N4JcuSbo6Uu3U5","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFmqGUq87N4JcuSbo6Uu3U5.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/FmqGUq87N4JcuSbo6Uu3U5.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"379bc669bbea5ddf1de53e795f79723b9559e9f2a54a0b429dc76f23442e00e5","result":{"confidence_high":91.324,"confidence_low":83.87599999999999,"sample_count":null,"score":87.6,"score_display":"87.6","source_stated_rank":70},"run_fingerprint":"25e385fe24d709a4f4b404730f60a8a21b2aefb38325f701497b121c2a209b27","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6e82e1189c72dd0e93b9d7ed","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e5f8e0e08c49e67f9bfc210fb452655004b2d93d60b3b19216301822daf1b84e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.00793,"date_is_proxy":true,"latency_seconds":39.204,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.343},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":128000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.5-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c836815ba52b5db805192d7fc6df98680a87a4e93d72ee62acd9f6a0f6328c92","result":{"confidence_high":86.58628,"confidence_low":85.24172,"sample_count":null,"score":85.914,"score_display":"85.9","source_stated_rank":52},"run_fingerprint":"8280bd1d1b5242ce24d1647125e3ea35a069711184c6be4003c39b5b36cc7d3c","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":245,"at":"2026-02-02T10:53:14.986Z","basis":"evaluation_started_at","evaluated_at":"2026-02-02T10:53:14.986Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"69f5c7823567437ab01417477896338578d5dc794deff0342abbec8c8cb10a29","metric_details":{"best_score_across_scorers":"0.922","model_release_date":"2026-01-27","stderr":3.9},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mbfZvrSNoXSJqCyD4VKqLe","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmbfZvrSNoXSJqCyD4VKqLe.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mbfZvrSNoXSJqCyD4VKqLe.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"53b5afd59ed2fcc1bea62c05d6126c9965d6161c4095ab8a5398b2b254a04a9f","result":{"confidence_high":99.84400000000001,"confidence_low":84.556,"sample_count":45,"score":92.2,"score_display":"92.2","source_stated_rank":62},"run_fingerprint":"8ef9f183062c53c7a0f305999cf530bb6f102bf16b31f96a080f0b6b26e36a14","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_04028ed53a8f5ae82875593a","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e35e7a23a006f85611411e94f8559dfa9bd09e95a736be9a4786594db7359486","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.28,"model_release_date":"2026-01-27T00:00:00.000Z","results_url":""},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"","model_type":"CoT","upstream_model_id":"kimi-k2.5"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d2f5a5a3807c2d47e07c18d7540fe3125ac78f0374c963dd93a0a538bb54bb38","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":11.81,"score_display":"12","source_stated_rank":142},"run_fingerprint":"abd1993e7ad4d9650e71766fd9ea52bd287093f5743fdb5d3b4b1f5b9746faca","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5231d77a85df897c20d6ce6e","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ce252117b3d6c05c7451a7f2a5917d274b93dd27855ef44e2ad57d433fbf589c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.28","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.5","source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"a85fe72f8380601a8993d6ae6a3cf7065c6eb262ddd38b8b87324249b59d0ba4","result":{"confidence_high":15.553805,"confidence_low":8.872649,"sample_count":360,"score":11.81,"score_display":"11.8","source_stated_rank":138},"run_fingerprint":"325f3fb1be62fc78c9c194c93a5a4462e629c485db98069e0e9b0caa9e17f736","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_484e3941e6a2ffc02d57d45f","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d12bb97154de5bf8b082ebf6db0227626237d6eb0edf4bc6f40a2744b35eaa5d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.019563,"date_is_proxy":true,"latency_seconds":110.011,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.874},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":128000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.5-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b8bbee82e883556ef6c402fdd018ce1adae629ba19386f8873bc5c3448148b92","result":{"confidence_high":86.04804,"confidence_low":82.62195999999999,"sample_count":null,"score":84.335,"score_display":"84.3","source_stated_rank":26},"run_fingerprint":"a019d8753453ecc73f04b6da9af3a2161ac2fd6373890163e8ccb0e0b5e35a14","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_1dbdadbee92bb7f366d1025b","split_or_window":"standard","unit":"percent","version":"MMMU-Pro official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":165,"at":"2026-04-23","basis":"evaluation_at","evaluated_at":"2026-04-23","first_observed_at":"2026-08-29T23:52:50Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"5b9065044d31da0a9ed5c8442959d0185d06bce612e8c872529d71d4b39fd758","metric_details":{"num_parameters":1026879376368,"pull_request":119,"result_file_url":"https://huggingface.co/moonshotai/Kimi-K2.5/resolve/refs%2Fpr%2F119/.eval_results/mmmu_pro.yaml","source":{"isExternal":false,"name":"Model Card","url":"https://huggingface.co/moonshotai/Kimi-K2.5"}},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"9225c08e521da63df44b0ad31230f50489ffbee930c8fb4aaf2f0f8dbf6fce3f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":78.5,"score_display":"78.5","source_stated_rank":3},"run_fingerprint":"5401e0f3e2ee071a9bdbcb47512f9c66778d9a616df98e48f514a2a1867d6abe","source":{"content_hash":"bc590a866070c9674cba0de4af7d92af6e0921bd2db419fca54624e00d73400b","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-10-05T12:33:31Z","homepage_url":"https://mmmu-benchmark.github.io/","id":"mmmu-pro","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"MMMU-Pro","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/api/datasets/MMMU/MMMU_Pro/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_abcf7008dfdcf21749bcd70b","provider_config":{"source_id":"mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"632664caf93c44f16ae6dfd6d365cf1b0e8afa8a7b996b9c1924f06a6d0aadc0","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.5 (Reasoning)","source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"d54638f4ca45fb6c067f5ce4c9ba2dd1e3cfac45b51ddfe7af5014674c9ecbb5","result":{"confidence_high":10.175118,"confidence_low":0.920927,"sample_count":71,"score":3.14285714285714,"score_display":"3.1","source_stated_rank":121},"run_fingerprint":"2b62bc4cdd91d7d1db393e66e413386eca228e9077f1d457bf75a7b2362360d5","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3274c202f27b162a812bd5b0","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"16bf0799603b2c0a007f349b6900a1f015b0e80a58161b963557266c2e147bc4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"56f2ca3e622b4d4d099aec16d36c9c43e7e45524034c416dc2bb838c50d8990e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.8,"score_display":"46.8","source_stated_rank":57},"run_fingerprint":"d8a906fc96cfb55277f407b385aeb50c6fbb3b7a2892e3af0348c2f0738b369c","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cdd77eeb27e64fe682104bb2","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":249,"at":"2026-01-28T17:12:03.604Z","basis":"evaluation_started_at","evaluated_at":"2026-01-28T17:12:03.604Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"48febe2e772dcb87452e53518815154bdcb4d974069763bb25194d1066851cf1","metric_details":{"best_score_across_scorers":"0.12","model_release_date":"2026-01-27","stderr":3.2659863237109046},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"f7vGNi73Cs5wSUsBMSzgFu","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Ff7vGNi73Cs5wSUsBMSzgFu.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/f7vGNi73Cs5wSUsBMSzgFu.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"61736386f6fbcc14ef55c41462b9ae590f28da972305a44e2a630610ced68ccc","result":{"confidence_high":18.401333194473374,"confidence_low":5.598666805526627,"sample_count":100,"score":12.0,"score_display":"12.0","source_stated_rank":127},"run_fingerprint":"bd3fa5ad8c69e92fe0cb605b1f0e0fa27f0b21c67b487cff0a425bcd30452120","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a1d127246f6c81742ed99c87","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"57182eea211c1e6e83522c915fcc1bd850edfce902d784a85c398e93bf834347","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.15","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":""},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.5","source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"cc43816fd6d881f3b38fb823a89342968180109a116a45d9f4de815ec4d9cf15","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":65.33,"score_display":"65.3","source_stated_rank":142},"run_fingerprint":"61e0cb69b5f86519e8ec02d4cb3394a6096a29bed2491be0ee482d2b9a57e7dc","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a605b31119223a617f1c6466","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"de21664b44b198f4d784248d86c235ff0decef98064e9523c2da26e0b05c0553","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.167669,"date_is_proxy":true,"latency_seconds":339.242,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.051},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":128000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.5-thinking"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"99b22638af7f34c252ae33003697fb22d97f158490750ed253cc3317fd9ce617","result":{"confidence_high":74.01996,"confidence_low":65.98004,"sample_count":500,"score":70.0,"score_display":"70.0","source_stated_rank":61},"run_fingerprint":"e172e8eb27024d8f26daa0b291e77e1b046367304b5c39e3310ece44cd57532f","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":229,"at":"2026-02-17T14:43:08.990Z","basis":"evaluation_started_at","evaluated_at":"2026-02-17T14:43:08.990Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"7b5146f4569aa4c47afec3eaf328b095409d61cb11235b3db0407e6ab4fed64e","metric_details":{"best_score_across_scorers":"0.737603305785124","model_release_date":"2026-01-27","stderr":2.0017832064928274},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"5wVjUb3yn6bdjPSfPEMuiZ","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5wVjUb3yn6bdjPSfPEMuiZ.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/5wVjUb3yn6bdjPSfPEMuiZ.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"cd5374ee246f175c7e7454192f83513308d0dcf956e848e5c7c486d2ce344e77","result":{"confidence_high":77.68382566323834,"confidence_low":69.83683549378645,"sample_count":484,"score":73.7603305785124,"score_display":"73.8","source_stated_rank":18},"run_fingerprint":"24868eb26355bc2f95324e7dc663edd356733ca6d8660be8679c81d0666ca019","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_21e5fc65c042eac9bcce3865","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":230,"at":"2026-02-17","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17","status":"stale"},"measurement_id":"57fc58a3aedf45a1e88d55a443ec7c2e0285eddc40087330e233322bc84d6fcd","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_kimi-k2-5-high","logs":null,"model_display":"Kimi K2.5","model_org":"Moonshot AI","reasoning_effort":"high","resolved":70.8,"site":"https://moonshotai.github.io/Kimi-K2/","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: kimi-k2.5","Org: Moonshot AI","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_kimi-k2-5-high/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"4bab498974efdc3ebd1bd27244930437d6d091484cdcbd6fef6d1e80ce3b5960","result":{"confidence_high":74.614817,"confidence_low":66.667998,"sample_count":500,"score":70.8,"score_display":"70.8","source_stated_rank":27},"run_fingerprint":"05f5ca71d17b34950b649c903ce44c30d444f689ff005a845fb668c6fce53158","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1ae656cb258a39c3338c0af1","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_2125f5e1cd1e1b1aa6168fb5","split_or_window":"2026-02-01/2026-03-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":218,"at":"2026-03-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"4d9ea716c84dbaaa7a43d5a830adfccf8586f5538ea064bcd73c7464cb7e5630","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-01-26","pass_at_5":70.17543859649122,"potential_contamination":false,"sem":0.6613467141065804},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"Kimi K2.5__tools","window_from":"2026-02-01","window_status":"historical-post-release","window_to":"2026-03-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"141d7a694df4e1ca70e967074cda149ad778755d6ce8798e08f8a094860fec90","result":{"confidence_high":59.74862051202985,"confidence_low":57.15614139273206,"sample_count":null,"score":58.452380952380956,"score_display":"58.45","source_stated_rank":null},"run_fingerprint":"5ebda0432145016aaefc28405cad973ad75bfa27af6115b9d4d89ca511b2c057","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_025533521cd219b3ae285dd7","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"cursorbench","metric":"score","name":"CursorBench","release_id":"release_585f86b619b5f18be3e1bcbb","split_or_window":"official-current-suite","unit":"percent","version":"CursorBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:17Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:17Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"eb10d9bb4df89d32243e05e56413157675e5e14f9a321bc83f8c7b4fb3254d9e","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.87","epoch_dataset_updated":"2026-09-21","model_release_date":"2026-01-27","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"steps_per_task":"30","tokens_per_task":"9446","upstream_source_url":"https://cursor.com/cursorbench"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi 2.5","source_effort":"","source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Cursor coding agent","tools_allowed":"Cursor coding-agent toolset"},"protocol_fingerprint":"029b9f8cf451748e0a6d8451889112782ebbe66658a7ded4994329f66ea12ca7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":31.900000000000002,"score_display":"31.9","source_stated_rank":74},"run_fingerprint":"e0483ccb828c7fbce148cce43af560dc1c24483b8b49d775a8959f4643c21da4","source":{"content_hash":"ebc02c78a769d769a5de11269b9e28b9b4d3aeb7801fa65a1082bba3531fe07b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/cursorbench","id":"epoch-cursorbench","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; CursorBench result redistribution permitted with attribution (2026-09-02)","locator":null,"name":"Epoch AI mirror · CursorBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/cursorbench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_42a10e2a982af332ffb7092b","provider_config":{"source_id":"epoch-cursorbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dfba89e23a1f2d96cdbdbdedc0417fe63b4d237fcc9797be8b0b5ad415fff821","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.038656,"date_is_proxy":true,"latency_seconds":200.023,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.03},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":128000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.5-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"759fb9c6f8d90202f8e82e48da1590104066f4bba0560da9d6cc9600bfff3c32","result":{"confidence_high":85.8868,"confidence_low":81.8492,"sample_count":null,"score":83.868,"score_display":"83.9","source_stated_rank":46},"run_fingerprint":"d14e14b08ba1874476b373983c1f7196b89c0b82b47735f4509b57f2e9c4e0fb","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b76bb6822e6a2cb6ad8c202108df646efff5599f9f5a7f44523ef0890c7ac8fa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.5 (Reasoning)","source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"9a751ce3717bf2e8fcd5e6795198ab13ebf865224c830224359510696dd32fa7","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.9583333333333,"score_display":"49.0","source_stated_rank":99},"run_fingerprint":"65bc3842f5db0d161d07a568e0a19389d55ce1a77c54d7d278aec5f38857292c","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1307685283e08685791dd3b9","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"a1018408486588f96493a7c2a47a66f4d561ee8cfdfa46ff4f6263c679f7878c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.047458","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"kimi-k2.5","source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"860d50f7d2ce508d6a774bf679003e9580848eac4e86b958ad49d04a68a96675","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":821.65,"score_display":"821.65","source_stated_rank":63},"run_fingerprint":"c416a1a7d5e1848a8a451a5ac49c53dc46c0d296a1cbc5f3e4499251a3be56d8","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4a115fb2a00cd4c6610fdf9c","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aaafa3623d3220425a251aa448d7049fb16be750dd6c8c3f1487a0b538fda069","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.879292,"date_is_proxy":true,"latency_seconds":2570.381,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.258},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.5-thinking"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"04433e22a3c2b3139fb4c4cb27ac7a54d258e26311d854f3bc00136ff83f635f","result":{"confidence_high":23.921680000000002,"confidence_low":11.15032,"sample_count":null,"score":17.536,"score_display":"17.5","source_stated_rank":81},"run_fingerprint":"0578cda2fa62ba37df2514d6e6427b7d62abd6e26f50c30f6700872c5e84bcf2","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"01056afbb3058b4dc79ec035b826d22f1ca99f65c3fad8c24abfaf210039238f","metric_details":{"license":"Modified MIT","variance":7.0013867655318895},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"kimi-k2.5-thinking","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e6b5033ece78816613a6d0390dff3d51ea72429d3fc57c28fd4bba3c6f10c9b8","result":{"confidence_high":1441.0177156615218,"confidence_low":1430.645533837519,"sample_count":20149,"score":1435.8316247495204,"score_display":"1436","source_stated_rank":74},"run_fingerprint":"db93be522f4e7c864dbc5d3a0c8f61a83634efff065793c50e08cc7b8e80bf14","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"fc55cedbc0281d9cbbca6cdf5965b1c053732bc9cbb9a0d94f634fe0cacabf78","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"e6d5214cc54f6c87325242520f06940cc7f2cd9a61c4363f6cfcae2dc7fb835f","result":{"confidence_high":1441.67,"confidence_low":1431.07,"sample_count":19042,"score":1436.37,"score_display":"1436.37","source_stated_rank":73},"run_fingerprint":"86d780b57f271ca284056ca9e9d5e15dea40721c9e40c58bf819b95371dd91a3","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4f6afc79c6931617e8c0dacf","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f4187928d9773f63de70ee44b36127c49c471c927317279a0615733cf3f9311b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k2.5_none","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"9f8e2aebd4260e3f70bf6133189ae8bd20a6380aecf5faa4058096ac07adcc4b","result":{"confidence_high":1417.93,"confidence_low":1393.38,"sample_count":3095,"score":1405.66,"score_display":"1405.66","source_stated_rank":82},"run_fingerprint":"06ff0e652e617947a3a760629496fc0f3349fcd73760796bd7224ef0d6491b34","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_ef5e09002026430a2939dc60","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"79f3918b3db66ee56c1f1e9fe804cd90a6e765b661f1b1c94800de907ba4104d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"944c42581b9cf19afddbc0b18f843bea0572518f388d8eba131a1bd77f675dfb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":45.6,"score_display":"45.6","source_stated_rank":89},"run_fingerprint":"df66922b91d5106b780dadd2595b3c6e2c8c90436ceefb202c46565143e33882","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4ba684af8be038b2c7303841","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":251,"at":"2026-01-27","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-27","status":"stale"},"measurement_id":"828b1d895f026a2062bb0d960f544cf652fcf61643a4d6398980c2ad07f3f542","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"standard_error_points":2.8777059907,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Kimi K2.5","source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":"2026-01-27","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"d8c5cba3bba6377e886940ccf395828c00454396e8e5cd22adf381e9b1bc42d9","result":{"confidence_high":48.842550932772,"confidence_low":37.561943449227996,"sample_count":89,"score":43.202247191,"score_display":"43.2","source_stated_rank":87},"run_fingerprint":"eb85aeeabdde9572f47f89b63cf0269108c85027d42234b65e6f59b07ece8a9f","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_243abd94b6a9f190e07fc408","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d83beaebdaeb9b3c4f8ac232507c3cdc370d527a30bea088e2b07cd523ffad79","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.085844,"date_is_proxy":true,"latency_seconds":1051.079,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.749},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":256000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":null,"top_p":0.95,"upstream_model_id":"kimi/kimi-k2.5-thinking"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"6e9cc71e85ab7809a3781f4f6c06da26508fd5f71e96c87843be1b1be98c92e2","result":{"confidence_high":43.41604,"confidence_low":40.47996,"sample_count":null,"score":41.948,"score_display":"41.9","source_stated_rank":67},"run_fingerprint":"88050949242a61dbfd8649e3b676e199d46c43e519c05b50b22c748aee569219","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"078cfacfed3a0ad234fca0f31c2d5945b98ec093725c130180608f30e478e8ac","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.292","mean_standard_error":"2.5","model_release_date":"2026-01-27","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.2,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.5","source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"b65a7f57075c73d3484361cc82f518acebb3dab3eb0fbf65d94886912ff4f1ef","result":{"confidence_high":18.712000000000003,"confidence_low":10.088000000000001,"sample_count":480,"score":14.400000000000002,"score_display":"14.4","source_stated_rank":47},"run_fingerprint":"dfdaa94d6a36efee92fc8a856df6da7779b140c7ecd432ae1a8e53e40e92cbac","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4a7613004fd4e192b71de9b9","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4df4dae28c3c67edc49bba11889271865930936505f327e7a49cc5eae3974fdb","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-01-27","notes":null,"upstream_source_url":null},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Kimi K2.5","source_effort":null,"source_model_version":"kimi-k2.5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"ceb47379ebb944853a74277a42f187b11105aff2cab403497864f58f2829b934","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":1198.4579999999999,"score_display":"1198.46","source_stated_rank":48},"run_fingerprint":"42edbf6b40ea6f767552b1da2533ef0661492fdbe9d8b6899db094fccf31ea13","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c40dc6af044b94a7c275bc96","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c90ce0cdd7a7cd47365fbe3e3bd7cbe51b95b447cc1d20faea18899687427245","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-01-27","reported_confidence_interval":"-27 / +27","source_locator":"Leaderboard models table · Kimi K2.5 (Reasoning) · gdpval"},"model":{"id":"moonshot/kimi-k2-5","name":"Kimi K2.5","provider":"Moonshot AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"kimi-k2-5","upstream_model_label":"Kimi K2.5 (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"d04b785d1f05285bd9b74a218c388948c28df55d52e3680fdcf43d65c40c25f2","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":824.06,"score_display":"824","source_stated_rank":null},"run_fingerprint":"a319e66719f93f14d32976813e0e1d04e64af7aaae91d582be73766f37d9c173","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Kimi K2.5 (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_6ae6ce92018cc9651626b3ec","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"529424b8df978dcff6551ea4489cc4d8aa2b62e29f5a5c05fdd1f8bdc0075aa6","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-16"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen 3.5 Plus (hosted 397B-A17B); model release: 2026-02-16","source_accessibility":"Unknown","source_model_name":"Qwen 3.5 Plus (hosted 397B-A17B)","source_model_release_date":"2026-02-16","source_model_versions":["qwen3.5-plus","qwen3.5-plus_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"91b57a1303ba601e13a5c7ba353c83076e4dda5988f1390cb7e609d675160383","result":{"confidence_high":148.08,"confidence_low":144.58,"sample_count":null,"score":146.79,"score_display":"146.79","source_stated_rank":67},"run_fingerprint":"e87d508ed2cca25163547391963a15a4fd8a51a52f41c88c2a2ac4898833513d","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_90e56fd70dd0364120d5d708","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"cbf57ac08c37c1b706b61bbe8a94eefecfce9935dc924daa74c0eeea239d1be1","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-13"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Qwen3.5 397B-A17B; model release: 2026-02-13","source_accessibility":"Open weights (unrestricted)","source_model_name":"Qwen3.5 397B-A17B","source_model_release_date":"2026-02-13","source_model_versions":["qwen3.5-397b-a17b","qwen3.5-397b-a17b_none"],"source_reasoning_efforts":["off"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"10f1b1093e2a715e82f34396a9a6a81638853bbf890fec33b5a27dbaea5bf88c","result":{"confidence_high":148.18,"confidence_low":144.82,"sample_count":null,"score":146.66,"score_display":"146.66","source_stated_rank":69},"run_fingerprint":"ccea31af3ec3ab5b133e1eb17a2e2242519e22ed05e548be00b2d8ec7984af77","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_90e56fd70dd0364120d5d708","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9b821ee4ba146dab0a221dc47e7ceab9fe0a2f55992dbc477d4ef3e45edf000e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-02-16","source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b","upstream_model_label":"Qwen3.5 397B A17B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"14b9f5618ffefe525eec2a647983d8ed5d7bd28231ff9cc3f9e353ebfbc42490","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":18.420425,"score_display":"18.4","source_stated_rank":null},"run_fingerprint":"d82f0779031657b626bfbfdbcd95e82bc2b538bc3a8afbccb978e5c789665c3d","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d177f1d7065fe8f8ce6ad0c2","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a122486acdff11fa097ee77ddcfaa409bad4e24bcfb12610414b8ddf66b4b412","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · intelligenceIndex"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b-non-reasoning","upstream_model_label":"Qwen3.5 397B A17B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"8d9580f4aa38e8f05ac7a47c09bc3ef9869d20d3b750588b68648e6b33fb8434","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":21.436655,"score_display":"21.4","source_stated_rank":null},"run_fingerprint":"380d47e262be079902eabc572b759b3fe8fea2e69c43ae421a67477941700b3b","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_f05cee9da83990f5cc4a1d0c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"594c77a79f7896053bc630e4b4a98f102a68f406a058b175f0625d237b0cb932","metric_details":{"license":"Apache 2.0","variance":2.5786167308169943},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1444.9599921779297,"confidence_low":1438.6653436616784,"sample_count":88660,"score":1441.8126679198042,"score_display":"1442","source_stated_rank":92},"run_fingerprint":"40a3927ff8600cbc27890dae36d1b6c80211869e2fde22dcbd588c7c2a1da6ef","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_062b1dd6a5a62c713b280490","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_2f3c6e21d0f3616fdb78f627","split_or_window":"full-multimodal-cot-no-tools","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":231,"at":"2026-02-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-16","status":"stale"},"measurement_id":"3e8d2522565e34e4fb8783d591b72fcb487cbf8d0a9623b5efb70d6cb9c7bd21","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2500 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-397B-A17B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE w/ CoT; Qwen3.5-397B-A17B column"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE with chain-of-thought evaluation; the card separately reports tool-augmented HLE; task count, judge, thinking configuration and run count not disclosed","question_count":2500,"reasoning_effort":"reported","result_published_at":"2026-02-16","source_content_hash_method":"raw_bytes","source_content_sha256":"bda4b24e975f65ac677985306ddbad33362e65064637660654c8bb7dea7edf50","source_published_at":"2026-02-16","tools":"none"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"868bb776b8522904ff0c5ee9dd97bd51db6e2af1f9b16da3811eb93464730240","result":{"confidence_high":30.504878,"confidence_low":26.960482,"sample_count":2500,"score":28.7,"score_display":"28.7","source_stated_rank":null},"run_fingerprint":"bcd5d7cdee647bb9f08d1813095c2bab2574d416311f8e80d993c7869682c8ea","source":{"content_hash":"3469425adbd0d60012104bab287e8f7f8ee7ce5bf5313511308a7d9b124fcd09","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-03T09:21:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B","id":"qwen3-5-397b-a17b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-397B-A17B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE w/ CoT; Qwen3.5-397B-A17B column","name":"Qwen3.5-397B-A17B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_28d4327ed17739786e2e7ac4","provider_config":{"source_id":"qwen3-5-397b-a17b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_287aa4bcb31b5b1ba26e562a","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"HLE · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"cafe86dcb3b01b30356e76575db2ad2aaafe8a08f2b2d213a8edb61f48d82861","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.5-397B-A17B column"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card HLE evaluation; task count, split, tools, judge, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"a84a031c29362410cc334384b07675f01d0dae9a081b10ca8c1b04573c33fbea","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":28.7,"score_display":"28.7","source_stated_rank":null},"run_fingerprint":"427c7a08323f03339730177911106c659fd643d061eff082d77111261ec4e285","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; STEM & Reasoning; HLE; Qwen3.5-397B-A17B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54abff30262a823e8f5a7fc3","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":231,"at":"2026-02-16","basis":"evaluation_at","evaluated_at":"2026-02-16","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"93b85b39e702b38053fbb9833f80a93a44f5c118464c7244cc0524b30bb6611e","metric_details":{"modality_lane":"not reported","notes":null,"num_parameters":403397928944,"pull_request":9,"result_file_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B/resolve/refs%2Fpr%2F9/.eval_results/hle.yaml","source":{"author":{"_id":"63e0eea7af523c37e5a77966","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1678663263366-63e0eea7af523c37e5a77966.jpeg","followerCount":485,"fullname":"Nathan Habib","isHf":true,"isHfAdmin":false,"isMod":false,"isPro":true,"isUserFollowing":false,"name":"SaylorTwift","type":"user"},"isExternal":false,"name":"Model Card","url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B"}},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":28.7,"score_display":"28.7","source_stated_rank":45},"run_fingerprint":"d9d6c7522b8785d3174c194da8d375365418c3eee4613059786332f55798cd9c","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6dfdfea19387039600fef367","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":231,"at":"2026-02-16","basis":"evaluation_at","evaluated_at":"2026-02-16","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"07d8f301b9537c7786790855f55d5f2acb8518e14cb8523ec9dc94c91e29bfc4","metric_details":{"modality_lane":"not reported","notes":"With tools","num_parameters":403397928944,"pull_request":9,"result_file_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B/resolve/refs%2Fpr%2F9/.eval_results/hle_with_tools.yaml","source":{"author":{"_id":"63e0eea7af523c37e5a77966","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1678663263366-63e0eea7af523c37e5a77966.jpeg","followerCount":485,"fullname":"Nathan Habib","isHf":true,"isHfAdmin":false,"isMod":false,"isPro":true,"isUserFollowing":false,"name":"SaylorTwift","type":"user"},"isExternal":false,"name":"Model Card","url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B"}},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.3,"score_display":"48.3","source_stated_rank":16},"run_fingerprint":"44a731da74d06b8ee5db410c9ce575c0f22ee94215bab2909e7d3a39110ef775","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6dfdfea19387039600fef367","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"aa3f8d06ac8e2980bf19993f310b432ec67510db2a30bd9d3a4cf84a206eab25","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-02-16","source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · hle"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b","upstream_model_label":"Qwen3.5 397B A17B (Reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"6679c077725daf7e716ac273916b11422fec2da5138da07dee2d0fd6f787443f","result":{"confidence_high":30.911822,"confidence_low":27.086951,"sample_count":2158,"score":28.962002,"score_display":"29.0","source_stated_rank":null},"run_fingerprint":"a251c30d3868e2ef531c8f7dc3c749c91b7b8d4af0a44341b145377240112ff9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d177f1d7065fe8f8ce6ad0c2","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5557049d619d88861029ae13f99b3009147481a74b681052b0aa6d7e88f3b8ef","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · hle"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b-non-reasoning","upstream_model_label":"Qwen3.5 397B A17B (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"92995ed512852dbc09db9f9decbcb3955abafa3a134fe2fdf034a1f9c12a1c8e","result":{"confidence_high":21.568524,"confidence_low":18.205047,"sample_count":2158,"score":19.833179,"score_display":"19.8","source_stated_rank":null},"run_fingerprint":"6bde0b9c0e9d61da9c017b384952a6151045cfdb21e702f31416d6a68c2a60fd","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_f05cee9da83990f5cc4a1d0c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5ac2e7e9a4ff78f40d82ab28b4f3d6e4ca1d333850924b1b77d29a839c37e9af","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-02-16","source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b","upstream_model_label":"Qwen3.5 397B A17B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"84ba1315ad5c1196d34fe0cea43299ffa63479b71fb7bb0ab1588ad60b09f251","result":{"confidence_high":92.875878,"confidence_low":84.214275,"sample_count":198,"score":89.292929,"score_display":"89.3","source_stated_rank":null},"run_fingerprint":"9383956a3906557626e577e82759f06e4ce9014d60756967c8c7bbc8783928af","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d177f1d7065fe8f8ce6ad0c2","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2ef23ea13f0d5a778313dcba4f577a05aded530522079487e8aa4f0cb57285c0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · gpqa"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b-non-reasoning","upstream_model_label":"Qwen3.5 397B A17B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"2bf7438afa84f9fe308a0a3e60976bb66907daeb107a5e31eb2758ee6e6302a3","result":{"confidence_high":90.201629,"confidence_low":80.546919,"sample_count":198,"score":86.060606,"score_display":"86.1","source_stated_rank":null},"run_fingerprint":"1c9dfb44a4ea49f40cab55610ae1e5fa3688eda83b8379f3c9292542cc5ea954","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_f05cee9da83990f5cc4a1d0c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:24:06.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:24:06.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dc81903196e54b40a60f758d66b814fb68cd381687bc36af70b6c8e0f6c7349e","metric_details":{"best_score_across_scorers":"0.8585858585858586","model_release_date":"2026-02-13","stderr":2.4825909793343355},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"gZMFRbVKnBmcnZU4YqqKMN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgZMFRbVKnBmcnZU4YqqKMN.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/gZMFRbVKnBmcnZU4YqqKMN.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"ae39be8c55d1929f388747056bfdd9fa55cb7d383fba0c5d87a2e6d83bdffdf2","result":{"confidence_high":90.72446417808115,"confidence_low":80.99270753909056,"sample_count":null,"score":85.85858585858585,"score_display":"85.9","source_stated_rank":82},"run_fingerprint":"d562655b0e1f2d9185772d0c8d4e3e8d2fae1072baac3a33497a57315265c79a","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f6988689a3282c76cf3fb1e9","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-07T00:24:02.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:24:02.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cc68318d6c8be3e673c18212bf0b357a07edffd86a5fd664a69f4f7dc1ca7a34","metric_details":{"best_score_across_scorers":"0.8636363636363636","model_release_date":"2026-02-13","stderr":2.445015597318985},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"bXrmQqzF9sXbuZfRrSc7NP","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbXrmQqzF9sXbuZfRrSc7NP.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/bXrmQqzF9sXbuZfRrSc7NP.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"0ce3e0a4976285f9be3ad330d4a7ee99d9b0538a09f461c10a08c7a7e46de3a7","result":{"confidence_high":91.15586693438158,"confidence_low":81.57140579289114,"sample_count":null,"score":86.36363636363636,"score_display":"86.4","source_stated_rank":76},"run_fingerprint":"8108a3f53e9dd1544b09be5fbb3c23724f3225c13135289e7569e95721932f3e","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_f05cee9da83990f5cc4a1d0c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_d7983b534cfcbf543e50af4a","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"GPQA Diamond · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"3e45857286fac380c5f708e944f6c758bf8e1c559f57ef87b8cb2b0365c294cc","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; STEM & Reasoning; GPQA Diamond; Qwen3.5-397B-A17B column"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":5,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card GPQA Diamond evaluation; prompt, task count, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"dd2cbcaba371ecefe249fd5325da514fd694c6b98d78339e083b881ae780a231","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":88.4,"score_display":"88.4","source_stated_rank":null},"run_fingerprint":"f54e99735714fbfccf1b9b0052ecff7a26f665efd6404d33d1b2961b16d6908d","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; STEM & Reasoning; GPQA Diamond; Qwen3.5-397B-A17B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54abff30262a823e8f5a7fc3","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_95dc04609d027072fa229d3b","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMLU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"ce207ba88dc13c7a99aa5a3d58b8d984d8e570abeabe617e299acfb27c5b711c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.5-397B-A17B column"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMLU-Pro evaluation; prompt, shots, task count, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"44f56864a490bb7582461036c068400061930b80aee3b634127a43235a3bd532","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.8,"score_display":"87.8","source_stated_rank":null},"run_fingerprint":"e3570f95190160986b5ad0e26740a4eac24c380aefbdd2a5ed46b3f4657f928f","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Knowledge; MMLU-Pro; Qwen3.5-397B-A17B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54abff30262a823e8f5a7fc3","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_95dc04609d027072fa229d3b","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMLU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":231,"at":"2026-02-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-16","status":"stale"},"measurement_id":"0f2d85680b6628429dbc3ad5fae0a3441ce4362f92b9b5ffb5b33099d3c16318","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-397B-A17B official revision-pinned model card; Language benchmark table; Knowledge; MMLU-Pro; Qwen3.5-397B-A17B column"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMLU-Pro evaluation; prompt, shots, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-16","source_content_hash_method":"raw_bytes","source_content_sha256":"bda4b24e975f65ac677985306ddbad33362e65064637660654c8bb7dea7edf50","source_published_at":"2026-02-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"4b3ce9472c40ddc49ea517f2999dfa6b5a85fb57fa96f8a6c4f2e5aaea297f84","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":87.8,"score_display":"87.8","source_stated_rank":null},"run_fingerprint":"019267dfe2133ae25c928fc89b53f7ba2298b7ba9a306469f479ed984d1a0fb4","source":{"content_hash":"3469425adbd0d60012104bab287e8f7f8ee7ce5bf5313511308a7d9b124fcd09","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-03T09:21:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B","id":"qwen3-5-397b-a17b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-397B-A17B official revision-pinned model card; Language benchmark table; Knowledge; MMLU-Pro; Qwen3.5-397B-A17B column","name":"Qwen3.5-397B-A17B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_28d4327ed17739786e2e7ac4","provider_config":{"source_id":"qwen3-5-397b-a17b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:12:18.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:12:18.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6d937779723f74e827fddc4506e52ab48b84589772c84f51320c1e62398701e6","metric_details":{"best_score_across_scorers":"0.8888888888888888","model_release_date":"2026-02-13","stderr":4.737793696791344},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"JYuFdyZZSiMvmSZNzMTywb","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJYuFdyZZSiMvmSZNzMTywb.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/JYuFdyZZSiMvmSZNzMTywb.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f767e31ef1f628d16486a10f8c816a461a1bf1fc45e3fd6d62bc0494285aef5a","result":{"confidence_high":98.17496453459992,"confidence_low":79.60281324317785,"sample_count":45,"score":88.88888888888889,"score_display":"88.9","source_stated_rank":71},"run_fingerprint":"be5cad1b1bba7609d37c8f05887fd74a54da16792ed6c75e8925e4f6759d17b9","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7cce63965b2039e8ed31c47c","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:12:04.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:12:04.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fd9ca23365ea89b8a130f17018450cea548dbfa5748692511942df6e90fca22e","metric_details":{"best_score_across_scorers":"0.8222222222222222","model_release_date":"2026-02-13","stderr":5.763774795025092},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"i2ZUgMREKiFAnE2M54w3Py","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fi2ZUgMREKiFAnE2M54w3Py.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/i2ZUgMREKiFAnE2M54w3Py.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f5a26e1f2584984e5b37a4593bfa772fb99969ad2597c0314d03fd2c53e09897","result":{"confidence_high":93.51922082047139,"confidence_low":70.92522362397304,"sample_count":45,"score":82.22222222222221,"score_display":"82.2","source_stated_rank":103},"run_fingerprint":"8ffcc5512f25b6962635816ae55d1edc23294ebdcb0580d4e4b4a24392cf19a8","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_f05cee9da83990f5cc4a1d0c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:34:18.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:34:18.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cd2153dedf13b4aaa9c15b55186d491c21532f3d0d80792aeb90d6dcd69f66f7","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.312280701754386","leaderboard_private_problem_count":285,"model_release_date":"2026-02-13","public_example_count":10,"stderr":2.7499133473298816},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"A4yUA9PfzRr92JAqDDo9ro","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FA4yUA9PfzRr92JAqDDo9ro.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/A4yUA9PfzRr92JAqDDo9ro.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.5 397B-A17B","thinking":null,"upstream_model_id":"qwen3.5-397b-a17b_none","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"d8ecfaeb0fafa1b6bb87324c051d31193978f4f8bbf35f3aa4df329071cebd42","result":{"confidence_high":36.61790033620517,"confidence_low":25.83824001467203,"sample_count":285,"score":31.2280701754386,"score_display":"31.2","source_stated_rank":69},"run_fingerprint":"a49de0ae3d828f5474568ac0d05fe0516271ad41144ab0bae7fb359217f3877d","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_f05cee9da83990f5cc4a1d0c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-28T12:28:11.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T12:28:11.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f81b5378ff698ee5a97ca586afc1f114b232b15d7a76aaac02be8a307b7d3a27","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.29473684210526313","leaderboard_private_problem_count":285,"model_release_date":"2026-02-13","public_example_count":10,"stderr":2.705413430767295},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"EzMaDdXs4YQTMLgppBGxAb","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEzMaDdXs4YQTMLgppBGxAb.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/EzMaDdXs4YQTMLgppBGxAb.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Qwen3.5 397B-A17B","thinking":null,"upstream_model_id":"qwen3.5-397b-a17b","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"dfb6930721dfbd0f9ed709f0a48ac6d73e816e7d4627556053a09b9ece2c66e0","result":{"confidence_high":34.77629453483021,"confidence_low":24.171073886222413,"sample_count":285,"score":29.47368421052631,"score_display":"29.5","source_stated_rank":71},"run_fingerprint":"f9e2a86d5619ec9a54b09fa0a648ae8b6823c6e4442308b492c33799a8b404b6","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c07028bf89b8c7098ca376ba","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8358f86980b57ab1db8170de4ffa2850cdae6038ba89577c285cd06d3e5203e2","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-02-16","source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b","upstream_model_label":"Qwen3.5 397B A17B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"5b9f2e39a677edcdc35796d76e9b20748acf1e870eb82b55b455618a4329dbfd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":77.283237,"score_display":"77.3","source_stated_rank":null},"run_fingerprint":"487c9ff6912b44cace56598e9417936901823e1623fab8c5f21ac76c89679665","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d177f1d7065fe8f8ce6ad0c2","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"621ff388691a7198cac7cf1a1e6350f91827ab6330d58096e79d8559e7a7cfb5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · mmmuPro"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b-non-reasoning","upstream_model_label":"Qwen3.5 397B A17B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"762f95639ac33890ed62bfd5044ef1eea90d6a1b9cfd4c9409faefabfc509f9c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.65896,"score_display":"52.7","source_stated_rank":null},"run_fingerprint":"0ce9862df12e841171c02b36c6f4f2afd4a18e535bc8650b185bfae13f35df19","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_f05cee9da83990f5cc4a1d0c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_79984934735c6e815678be85","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"MMMU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"bde251e1e0bf461ee90aedd05766bf18cda4f5cd75a268e8077f9147e49b6345","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.5-397B-A17B column"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMMU-Pro evaluation; prompt, task count, tools, reasoning mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"56867732d9f2c0f7b77304c985a2db6d9d0d33fe8e7e4c87aca1036339e4fe4b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.0,"score_display":"79.0","source_stated_rank":null},"run_fingerprint":"5b8b72fa34d94879879976b75b62b4a9f1d83b6ec1def2506e490ceebf19cdd0","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Vision Language benchmark table; STEM & Puzzle; MMMU-Pro; Qwen3.5-397B-A17B column","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54abff30262a823e8f5a7fc3","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_d28d13464cb7d9d7990aac7d","split_or_window":"standard-vendor-protocol-undisclosed","unit":"percent","version":"MMMU-Pro · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":231,"at":"2026-02-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-16","status":"stale"},"measurement_id":"a0fc4306070f7aedf6e773207b4bb69c741fe14da6abe3640d3b5432d0f12629","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-397B-A17B official revision-pinned model card; Vision-Language benchmark table; STEM and Puzzle; MMMU-Pro; Qwen3.5-397B-A17B column"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card MMMU-Pro evaluation; prompt, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-16","source_content_hash_method":"raw_bytes","source_content_sha256":"bda4b24e975f65ac677985306ddbad33362e65064637660654c8bb7dea7edf50","source_published_at":"2026-02-16","tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"50a715370ebf78106e3f1db2cddc95a3047c6c0c6380a22d9b82d0b92ae9c790","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":79.0,"score_display":"79.0","source_stated_rank":null},"run_fingerprint":"7ff01a611af536cb0c3edcde9f26c9db3759fd6a83c46eb079cce8d9009feabe","source":{"content_hash":"3469425adbd0d60012104bab287e8f7f8ee7ce5bf5313511308a7d9b124fcd09","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-03T09:21:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B","id":"qwen3-5-397b-a17b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-397B-A17B official revision-pinned model card; Vision-Language benchmark table; STEM and Puzzle; MMMU-Pro; Qwen3.5-397B-A17B column","name":"Qwen3.5-397B-A17B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_28d4327ed17739786e2e7ac4","provider_config":{"source_id":"qwen3-5-397b-a17b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"41b7ce44cca84b5139d8e5535cd7d982ae5eb53e30365fe362709dbf4cf76f1d","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-02-16","source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · critpt"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b","upstream_model_label":"Qwen3.5 397B A17B (Reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"0ed22ebee94af4e1241ccedef76d63297b5609736d959205f92986178828e1b4","result":{"confidence_high":8.096389,"confidence_low":0.340821,"sample_count":70,"score":1.706122,"score_display":"1.7","source_stated_rank":null},"run_fingerprint":"615dc157e0da4c40c43f4a6b0c6f743aa98b27634085d21741e51ce2e08dc597","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d177f1d7065fe8f8ce6ad0c2","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"cd3190f7b6a37afa73f1a11528f55139526e79b5841da531ad16adc920dd302b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":false,"model_release_date":null,"source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · critpt"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b-non-reasoning","upstream_model_label":"Qwen3.5 397B A17B (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"31d9007c4a204c75790b2913ad0eebaeb76082601cfef8f9233a92c4fb230f2b","result":{"confidence_high":6.724008,"confidence_low":0.10358,"sample_count":70,"score":0.857143,"score_display":"0.9","source_stated_rank":null},"run_fingerprint":"49e7fc202002059297a331f738aecd1b3d94b83bd4102ace3550c5f0ef1a3ae9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_f05cee9da83990f5cc4a1d0c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:28:16.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:28:16.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"23af96d0af161b62515a4e8625d3ebd1187d322e2f16df18da7acfb73ea0289d","metric_details":{"best_score_across_scorers":"0.12","model_release_date":"2026-02-13","stderr":3.2659863237109037},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"dagNWyu57ksu7gB7Q86eCx","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdagNWyu57ksu7gB7Q86eCx.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/dagNWyu57ksu7gB7Q86eCx.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"bf8d657efbbb79b8eb2f6088bbc6e245709608722acf5532f85720868ef7fbed","result":{"confidence_high":18.40133319447337,"confidence_low":5.598666805526629,"sample_count":100,"score":12.0,"score_display":"12.0","source_stated_rank":127},"run_fingerprint":"91ffb0ba04a802285f6d36f418baa13ca5bb60a3ec24c5616632b979b6de5d3c","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_00b9738ee0fe161310633159","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-07T00:27:56.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-07T00:27:56.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5a7bd3a104c4950f9181056d0b902281b6bde503fc85bf68d1958272dd0cf417","metric_details":{"best_score_across_scorers":"0.13","model_release_date":"2026-02-13","stderr":3.379976689896311},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"VRLKwMA8hJnNRpp8M3WDCw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVRLKwMA8hJnNRpp8M3WDCw.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/VRLKwMA8hJnNRpp8M3WDCw.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3d695709b95ab3768a4eae61fc76c99ef14abedd114d679239d3b05626701bd8","result":{"confidence_high":19.62475431219677,"confidence_low":6.375245687803231,"sample_count":100,"score":13.0,"score_display":"13.0","source_stated_rank":120},"run_fingerprint":"9605d05b878c12e3ea98db7793ae91c0de073f91b299d6cd2b01f4c7feed6625","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_f05cee9da83990f5cc4a1d0c","provider_config":{"source_label":"off"},"provider_mode_key":"none","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_9162beb27813d52a46a20a9f","split_or_window":"qwen-internal-agent-run","unit":"percent","version":"SWE-bench Verified · Qwen vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"96c3d27aadd523c2bbdb530cd5da43a329116dd83a0c30a0a97d2861c1c2c925","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.5-397B-A17B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Verified; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"76ed278b99ffd1e5f9230954bc4ee5268b74c5c42a740c706366f11a530ecbf6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.2,"score_display":"76.2","source_stated_rank":null},"run_fingerprint":"c2f707b99a2815e07c1030e5d1d1d5b3c12a1582bccd8136cfdf4c6caed5432f","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Verified; Qwen3.5-397B-A17B column; SWE-Bench Series methodology note","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54abff30262a823e8f5a7fc3","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_73b4dac2493679abaf562468","split_or_window":"qwen-vendor-protocol-undisclosed","unit":"percent","version":"SWE-bench Verified · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":231,"at":"2026-02-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-16","status":"stale"},"measurement_id":"ec701082748d35e1c7648d5eaebb27e2cfe183b6c2cc4a221166235b633eb51c","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-397B-A17B official revision-pinned model card; Language benchmark table; Coding; SWE-bench Verified; Qwen3.5-397B-A17B column"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card SWE-bench Verified evaluation; task count, scaffold, tools, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-16","source_content_hash_method":"raw_bytes","source_content_sha256":"bda4b24e975f65ac677985306ddbad33362e65064637660654c8bb7dea7edf50","source_published_at":"2026-02-16","tools":"coding-agent tools"},"run_count":null,"scaffold":"Qwen vendor agent scaffold not disclosed","tools_allowed":"coding-agent tools"},"protocol_fingerprint":"03be66369807dacab823ca80913edd18c4fed27ba1f7a056b4838c27e4c22fdc","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":76.4,"score_display":"76.4","source_stated_rank":null},"run_fingerprint":"0ab8ab3e31907c797b7ec0c4f25fc5c09d62ef9c221553a031125195f4381f2a","source":{"content_hash":"3469425adbd0d60012104bab287e8f7f8ee7ce5bf5313511308a7d9b124fcd09","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-03T09:21:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B","id":"qwen3-5-397b-a17b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-397B-A17B official revision-pinned model card; Language benchmark table; Coding; SWE-bench Verified; Qwen3.5-397B-A17B column","name":"Qwen3.5-397B-A17B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_28d4327ed17739786e2e7ac4","provider_config":{"source_id":"qwen3-5-397b-a17b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_7449132b5f078c71f23c26b5","split_or_window":"2026-02-15/2026-03-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":218,"at":"2026-03-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"b359c40930a830e9d1647d42947ed8fd8e385004370531197e9b0b3458cf1a15","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-02-15","pass_at_5":66.66666666666666,"potential_contamination":false,"sem":1.767759778631177},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"Qwen3.5-397B-A17B__tools","window_from":"2026-02-15","window_status":"historical-post-release","window_to":"2026-03-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"01da0eb2fd9c383f12ef300cd69d69d04eda6900ff1aa1003fe2c725832a2382","result":{"confidence_high":60.30241600372394,"confidence_low":53.37279767148973,"sample_count":null,"score":56.837606837606835,"score_display":"56.84","source_stated_rank":null},"run_fingerprint":"d5f0bf44ff1ea49a9ecdca3047d6021cb335d747e2dd7a384178604ea738d8f3","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ddb434ba4c54ecfb0e035efd","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-pro-public","metric":"resolved_rate","name":"SWE-bench Pro Public · Scale","release_id":"release_d4a5abde3ed9874198f1dc1f","split_or_window":"qwen-refined-vendor-run","unit":"percent","version":"SWE-bench Pro · Qwen refined vendor protocol"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"ca92f35681fbca190e514b31e33d58a1284433ac210f169c3d56b6ec0b84e900","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.5-397B-A17B column; SWE-Bench Series methodology note"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"SWE-bench Pro with Qwen's documented corrections/refinements to problematic tasks; internal agent scaffold with Bash and file-edit tools; task count, reasoning mode and run count not disclosed; temperature 1.0; top_p 0.95; 200K context","reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"Bash and file-edit tools"},"run_count":null,"scaffold":"Qwen internal agent scaffold","tools_allowed":"Bash and file-edit tools"},"protocol_fingerprint":"6026abb572c5a53a08110c13a15c48d68315673ae81f11df6bff8d0515dc915b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.9,"score_display":"50.9","source_stated_rank":null},"run_fingerprint":"0a26d6488b3fa476ce11fb60518533d6422994592f7265b430506926a22145ae","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SWE-bench Pro; Qwen3.5-397B-A17B column; SWE-Bench Series methodology note","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54abff30262a823e8f5a7fc3","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"4b99c9a407a4a753bb48bd7bbba59a00c5b66ca4d1cd20bb9c0f788a1c50ceb8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-02-16","source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · scicode"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b","upstream_model_label":"Qwen3.5 397B A17B (Reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"8a6aa5f4b195c63fb85c875f7a9506baa0e9bff924fc5a4743cf2abd6e772a68","result":{"confidence_high":50.566002,"confidence_low":39.154449,"sample_count":288,"score":44.791667,"score_display":"44.8","source_stated_rank":null},"run_fingerprint":"0701c818f38b8f7c1509df3f2fe0f2e6f2dcb5e36b5aaf321614bc2476450507","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d177f1d7065fe8f8ce6ad0c2","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0e9b52b7f05408079a2e3577ce5d87893db546d785ce9b660c701e7ab5148453","metric_details":{"license":"Apache 2.0","variance":6.780102320577902},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"qwen3.5-397b-a17b","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"661fc781baca7c783aaad4426dbb098e1aa262eab788294c696ca9a755162311","result":{"confidence_high":1404.5451278846726,"confidence_low":1394.338172635512,"sample_count":21280,"score":1399.441650260092,"score_display":"1399","source_stated_rank":86},"run_fingerprint":"531e54241fa1cf131e07c44c2b47c19d48b46a4274042b0167509cdbf69f836c","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4837b3098d8faed90802985c","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"e096fc50befd2dacda765b0481c8636b3f38b2b25838878142ddee2128f0fe73","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-02-16","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · terminalbenchV21"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"qwen3-5-397b-a17b","upstream_model_label":"Qwen3.5 397B A17B (Reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"5090124455d3b3f54bb1f18df5d34a639e2f8f85ae4e0dd765bf2762c2dcdca0","result":{"confidence_high":61.424061,"confidence_low":41.08918,"sample_count":89,"score":51.310861,"score_display":"51.3","source_stated_rank":null},"run_fingerprint":"d8c7c5e7b3561d7ec1e09c0b0860047b027c61e8625630d2b3e8ab763f4b0f96","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d177f1d7065fe8f8ce6ad0c2","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":220,"at":"2026-02-27","basis":"evaluation_at","evaluated_at":"2026-02-27","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"774f2c89b41f59cbfad5c4d294276cb25da44f4fc6e254bc7aa3d5e36669b67c","metric_details":{"cost":null,"pass_2":6.36,"pass_3":5.41,"pass_4":5.15},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"text-emb-3-large","submission_slug":"qwen3.5-397b-a17b-think_sierra_2026-03-02","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"f060008efbd1330fcb16847cd355ebaffc070cbb0b9dd1b31485f1394c535163","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":9.79,"score_display":"9.8","source_stated_rank":null},"run_fingerprint":"fe84073aebc890212cec32353097968c7079087435c05f3c78fceda66358e050","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fec2e8c4ae6e264376e8e9fd","provider_config":{"source_id":"tau3-bench","source_label":"enabled"},"provider_mode_key":"enabled","raw_label":"enabled","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"78420583027f97631a7aec78170f919ed5a8f7757c37b6f5809da0b73f96a50b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-02-16","source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · tauBanking"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"qwen3-5-397b-a17b","upstream_model_label":"Qwen3.5 397B A17B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"4d611e767498507525ef4da586c2f6c746037928a88d88bbd62e314fecb4642a","result":{"confidence_high":21.590168,"confidence_low":8.002381,"sample_count":97,"score":13.402062,"score_display":"13.4","source_stated_rank":null},"run_fingerprint":"125f895e25cf7944f5fb60b14ef5c038b475e1703eb2ef21713f0a9fc80aa55a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d177f1d7065fe8f8ce6ad0c2","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_5af9fe5a9d5e275323798c8b","split_or_window":"self-contained-78-no-api-subset","unit":"percent","version":"SkillsBench · Qwen 78-task self-contained subset"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":166,"at":"2026-04-22","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-04-22","status":"stale"},"measurement_id":"bdfed38e704e0c0b9349070d11f6a7a744e5b63f20f7e377d82e0cd62ae74244","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 78 scored items with a 95 % Wilson interval.","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.5-397B-A17B column; SkillsBench methodology note"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen vendor evaluation through OpenCode on 78 self-contained tasks, excluding API-dependent tasks; average of five runs; exact tool policy and reasoning mode not disclosed","question_count":78,"reasoning_effort":"reported","result_published_at":"2026-04-22","source_content_hash_method":"raw_bytes","source_content_sha256":"bb936d6da51014f1edc9aa4cf9abf28d98695b7616ad56adfeeebfa752051d3d","source_published_at":"2026-04-22","tools":"not reported"},"run_count":5,"scaffold":"OpenCode","tools_allowed":"not reported"},"protocol_fingerprint":"0075a32b62b602932d1572b58a2560fe97ac92d58166e3cd33e67ab077a6bac5","result":{"confidence_high":40.911461,"confidence_low":20.966118,"sample_count":78,"score":30.0,"score_display":"30.0","source_stated_rank":null},"run_fingerprint":"f4fe44d781b8822dc371f22076a32f5ac304b518eea29f0ca81e0e11c7677aca","source":{"content_hash":"f300e103f2543a27f62447e8f0f66cc289d5ae5a517ddadb55e8a3f85f8e89a3","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-05T16:42:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.6-27B","id":"qwen3-6-27b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.6-27B official revision-pinned model card; Language benchmark table; Coding Agent; SkillsBench Avg5; Qwen3.5-397B-A17B column; SkillsBench methodology note","name":"Qwen3.6-27B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.6-27B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_54abff30262a823e8f5a7fc3","provider_config":{"source_id":"qwen3-6-27b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:14Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:14Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"1e76bdb0adc610b65eef654b782d917a9a728075a4bd8a6ed918bc6431a96eee","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","mean_score":"0.40299999999999997","mean_standard_error":"4.6","model_release_date":"2026-02-13","notes":null,"standard_error_points":470.0,"upstream_source_url":null},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Qwen3.5","source_effort":null,"source_model_version":"qwen3.5-397b-a17b","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"a0791c18c9dadc0bc147af83d9b5e61e05188262494ba1fb587282160ca11083","result":{"confidence_high":100.0,"confidence_low":0.0,"sample_count":480,"score":24.9,"score_display":"24.9","source_stated_rank":39},"run_fingerprint":"592486eece74bbaded4f3de359062169ec9e535ed732347182d410e13bdb121f","source":{"content_hash":"124b097999c50b3a4383b24f06b5cbfb24916d22fb182f680202f9a2ed498437","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f2df9bdc5ed37041104d582e","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_6b9456463dae72f95fadf6b3","split_or_window":"vendor-card-protocol-undisclosed","unit":"percent","version":"BFCL V4 · Qwen vendor run"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":231,"at":"2026-02-16","basis":"result_published_at","evaluated_at":null,"first_observed_at":"2026-09-01T00:20:18Z","last_confirmed_at":"2026-10-04T14:29:19Z","observed_at":"2026-10-04T14:29:19Z","source_freshness":"current","source_observation_age_hours":23.94,"source_observation_status":"current","source_published_at":"2026-02-16","status":"stale"},"measurement_id":"83dd3b78c54352e64b8bff8bd90b81c57ef3bfc1d9658f94cec1b90bebdd251f","metric_details":{"comparison_warning":"The report date is used as a freshness proxy; no separate evaluation date was published.","comparison_warning_code":"report-date-proxy","evidence_verified":true,"extraction_method":"curated factual transcription from the linked primary report","source_locator":"Qwen3.5-397B-A17B official revision-pinned model card; Language benchmark table; General Agent; BFCL-V4; Qwen3.5-397B-A17B column"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"report-date-proxy","judge":null,"run_config":{"curated_transcription":true,"evaluator":"Qwen","evidence_verified":true,"freshness_proxy":true,"protocol":"Qwen model-card BFCL-V4 evaluation; category mix, prompt, thinking mode and run count not disclosed","reasoning_effort":"reported","result_published_at":"2026-02-16","source_content_hash_method":"raw_bytes","source_content_sha256":"bda4b24e975f65ac677985306ddbad33362e65064637660654c8bb7dea7edf50","source_published_at":"2026-02-16","tools":"function-calling tools"},"run_count":null,"scaffold":null,"tools_allowed":"function-calling tools"},"protocol_fingerprint":"2be3c8fde04bf33c6b6db13a6ae1981f288300b9baf0dde2bfae161e55ae4181","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":72.9,"score_display":"72.9","source_stated_rank":null},"run_fingerprint":"d7cb6a43287918f300482b7aabb4124578f3dff767b00503545c975d2b0a610d","source":{"content_hash":"3469425adbd0d60012104bab287e8f7f8ee7ce5bf5313511308a7d9b124fcd09","fetched_at":"2026-10-04T14:29:19Z","first_fetched_at":"2026-09-03T09:21:24Z","homepage_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B","id":"qwen3-5-397b-a17b-card","last_fetched_at":"2026-10-04T14:29:19Z","license":"Apache-2.0 model card; aggregate score facts with attribution","locator":"Qwen3.5-397B-A17B official revision-pinned model card; Language benchmark table; General Agent; BFCL-V4; Qwen3.5-397B-A17B column","name":"Qwen3.5-397B-A17B official revision-pinned model card","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_28d4327ed17739786e2e7ac4","provider_config":{"source_id":"qwen3-5-397b-a17b-card","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"result_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":2,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-model-vendor-report","verified_status":"official-model-vendor-report"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c49086e5dea9b609f487a3d26226a805728e35e7da1f41af9e567ef286afdcb0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":262144,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-02-16","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · gdpval"},"model":{"id":"alibaba/qwen3-5-397b-a17b","name":"Qwen 3.5 397B-A17B","provider":"Alibaba","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"qwen3-5-397b-a17b","upstream_model_label":"Qwen3.5 397B A17B (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"14a3c9d72360039f8b5f919009c7499bb1d78c9e7cd16a2f67a57e5ecfa7fc70","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":779.56,"score_display":"780","source_stated_rank":null},"run_fingerprint":"c6326e906a5fde3bac1d5163d992e3f15f3fa51196a246a2e33ae3ec9d607777","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Qwen3.5 397B A17B (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_d177f1d7065fe8f8ce6ad0c2","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"53001d158373d36fb3afbe9de87f3478f93f38fa391af91bf677798225afbd3e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · MiMo-V2.6-Flash · intelligenceIndex"},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mimo-v2-6-flash","upstream_model_label":"MiMo-V2.6-Flash"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"65d5558378e6ecd09170d348a5bbdaa6bfe4f22c2157b8dd4fab86b0a061a448","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":37.884359,"score_display":"37.9","source_stated_rank":null},"run_fingerprint":"59000be454c5c586e3d27e19a39246db715ccf45c2c0e23ac860f946204a3acd","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Flash · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85514b3a25f5a223df6b29bb","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"218865be56dcaaba9558e25429bdf8767b1deb7ca33a566bf8399d1ccff8f163","metric_details":{"license":"MIT","variance":16.007413959437955},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1459.6245338377475,"confidence_low":1443.941189608791,"sample_count":6053,"score":1451.7828617232694,"score_display":"1452","source_stated_rank":77},"run_fingerprint":"623865c7c8ecd5bce2cf128de0ad1e2af9b6dc9c4b3e5cb60c2b750d00a68543","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_739d929af0ee1a5fc9f85891","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5626c88a0af0827f3e637cb88d4e44079018e92c6c4fdc7a1b679108f48946a2","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · MiMo-V2.6-Flash · hle"},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mimo-v2-6-flash","upstream_model_label":"MiMo-V2.6-Flash"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"a017e0569a7bc94da62a38d17b7f3343292fdcd579c17b66514f7c6f041f35c6","result":{"confidence_high":37.117151,"confidence_low":33.093434,"sample_count":2158,"score":35.078777,"score_display":"35.1","source_stated_rank":null},"run_fingerprint":"257975a18074d6f73aa647489bf33bef06d83933c7bab7af2d6409e24e587f57","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Flash · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85514b3a25f5a223df6b29bb","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"2eab8d8cbd2fc82d4d47ba8f7ba8f238f0aa3c64a3e36c7327c85a349fbf993a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · MiMo-V2.6-Flash · mmmuPro"},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mimo-v2-6-flash","upstream_model_label":"MiMo-V2.6-Flash"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"aece27f0893777401d2a7cc575f839f6559b6bf25286d901cc0b2c34a4a14544","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.063584,"score_display":"73.1","source_stated_rank":null},"run_fingerprint":"5238b8de4e4d77a6a986f7d40b2abe8f49afe0d76e78c0af4daafb48b396d583","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Flash · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85514b3a25f5a223df6b29bb","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b727ba9f469aac4a0c71012b1de08b8df465f8dbbe1b167bc899812582eb3c40","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · MiMo-V2.6-Flash · critpt"},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mimo-v2-6-flash","upstream_model_label":"MiMo-V2.6-Flash"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"dbcac17b835f0fb828f8991210b3a24962714a44805986124b8e7b5f15674abb","result":{"confidence_high":21.648098,"confidence_low":6.305793,"sample_count":70,"score":12.0,"score_display":"12.0","source_stated_rank":null},"run_fingerprint":"acae5f55fe7163c461ea6286450d9726a9bef44c771e2147f0a800e12d032305","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Flash · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85514b3a25f5a223df6b29bb","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ioi","metric":"score","name":"IOI","release_id":"release_e9a397f5dc7f7ff80be0a322","split_or_window":"vals-ioi-2024-2026-overall","unit":"percent","version":"Vals IOI v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f7e411b7afc3bb2486cf4c59c9be45d13696dbd9bcd9e82fdf6810b3409ae5a0","metric_details":{"benchmark_page_updated":"2026-10-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.215104,"date_is_proxy":true,"latency_seconds":6521.606,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.785},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","max_output_tokens":128000,"protocol":"Vals AI independent run over IOI 2024, 2025 and 2026 problems; overall score (Fassung 2 seit 2026-09-10)","reasoning":null,"reasoning_effort":null,"source_content_sha256":"921c36395530ca2da7dd18b33b12992cdda44f65de2ab8c989755874c188b754","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.6-flash"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against olympiad graders"},"protocol_fingerprint":"8f5d37845af80fbc9db91e52bc607f42c8ca5cd052db46d726459493024e842d","result":{"confidence_high":57.0456,"confidence_low":38.2884,"sample_count":null,"score":47.667,"score_display":"47.7","source_stated_rank":25},"run_fingerprint":"4228b34e247878ccdd3f44e3a617baeadc919a165bf2074a7211a247d4bba54b","source":{"content_hash":"013261ec03c079368b0bfd30f02a7b25f93d5f191da2c0bfd4701e69df398236","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/ioi","id":"vals-ioi","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · IOI","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/ioi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_776318aa95331442d0fd0253","provider_config":{"source_id":"vals-ioi","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9b8504dd7fbf0e21458a99b041337a4e8aff9cbd47b2f1fc25ea42d633135c77","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","source_locator":"Leaderboard models table · MiMo-V2.6-Flash · scicode"},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mimo-v2-6-flash","upstream_model_label":"MiMo-V2.6-Flash"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"a56dd54118db691c325818c0c9466f6bbb7b6106c14b66ea9f505728fa319eba","result":{"confidence_high":56.991126,"confidence_low":45.521652,"sample_count":288,"score":51.273148,"score_display":"51.3","source_stated_rank":null},"run_fingerprint":"7ec4f6c409f74a7631cfec84102e9256bedb168c833d10a63ee35f44a7b4af45","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Flash · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85514b3a25f5a223df6b29bb","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"eb30fd6709a80671085f0240351197c13312a237e93b209ab8aa7634f0e4641b","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.563041,"date_is_proxy":true,"latency_seconds":3013.012,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.045},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":128000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.6-flash"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"75062034536814504b31e658ee66e2d582995d77fd7638dae5bf80cf9707328f","result":{"confidence_high":86.8882,"confidence_low":71.03179999999999,"sample_count":null,"score":78.96,"score_display":"79.0","source_stated_rank":23},"run_fingerprint":"f8684931d575a3c6f33bdbbf88e0934c08c7d20eaa2577bc8dc2d779731cd7ba","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8ec17d78c34548ea7a909552","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0c4e6d66ecb5a50084fb0db10a86971004463cf1cccc45b1359d740e9f06fb49","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.02332,"date_is_proxy":true,"latency_seconds":809.733,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.716},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":128000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":0.95,"upstream_model_id":"xiaomi/mimo-v2.6-flash"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"7ba4a2e6fbd7e7328cbfdd901656cb86570c72479fd031e98594c6a566f5b5ad","result":{"confidence_high":79.76736,"confidence_low":73.04064,"sample_count":null,"score":76.404,"score_display":"76.4","source_stated_rank":16},"run_fingerprint":"504b89a265de464986a2a53a07738f9078da2b67a069daf3550c8e50f408ce81","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8e264e308d488f525e186738","provider_config":{"source_id":"vals-terminal-bench-2-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"f01444f01ecbd4ac835aee90f03bb598d871d2bf20fe7b8cc3c88ae56c2ac454","metric_details":{"confidence_level":0.95,"license":"MIT","session_count":15279},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"MiMo V2.6 Flash","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1c1b7bf26f2d21cbb0b79ac729631ef0225250e306ff702e6f3ae42e40ad06c9","result":{"confidence_high":0.007153219502814034,"confidence_low":-0.018505760059126415,"sample_count":1562757,"score":-0.005676270278156191,"score_display":"-0.0057","source_stated_rank":33},"run_fingerprint":"9060ac95deeca7004eb74cedcbc283d6fa72bbcb91863bd60eedb345aa3a9a73","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_424a25c5919bec60800d439d","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5bd176833d9494efe7d6b419cc92ba506d627110ff19484c2fa3d6215a9f25ac","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-09-21","reported_confidence_interval":"-19 / +19","source_locator":"Leaderboard models table · MiMo-V2.6-Flash · gdpval"},"model":{"id":"xiaomi/mimo-v2-6-flash","name":"MiMo V2.6 Flash","provider":"Xiaomi","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"mimo-v2-6-flash","upstream_model_label":"MiMo-V2.6-Flash"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"42936a767fbffcb213a50d733ae2e784b09171b6202f9942c2ee8d32a751670d","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":1600.06,"score_display":"1600","source_stated_rank":null},"run_fingerprint":"f7d4cf510ac8d119330d64f6fbcf90a9e2365d35836767c9edf2662d6c8565a2","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · MiMo-V2.6-Flash · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85514b3a25f5a223df6b29bb","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8c9cde0ef5237b565913ba13051ce253d1a089c07ae3e6da8793ea3686a5d257","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-02-11"},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":false,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GLM-5; model release: 2026-02-11","source_accessibility":"Open weights (unrestricted)","source_model_name":"GLM-5","source_model_release_date":"2026-02-11","source_model_versions":["glm-5"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4142808fb98b12cf32b6751a91504e1a2529bd6d6b7b9e03e09ec5996c684bc2","result":{"confidence_high":147.68,"confidence_low":143.88,"sample_count":null,"score":145.84,"score_display":"145.84","source_stated_rank":77},"run_fingerprint":"4b99f9f72f1b46fa85d44da5ccf12b2869d02194743ee038e2805ef6bb6f6b48","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85a8ec980f1ad054744d25c6","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8d0e2481868906d699945c08c079a04eff51fde7a646bbc8f4d63cd8d114683c","metric_details":{"license":"MIT","variance":4.707434489305806},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1461.8643775499806,"confidence_low":1453.3594576228365,"sample_count":29174,"score":1457.6119175864085,"score_display":"1458","source_stated_rank":63},"run_fingerprint":"acaf6d55cca5b8c9072d771578cb601df791e68713a66cce64801fedfb29e6a3","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0f408aa2a737e269f251bd52","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":234,"at":"2026-02-13","basis":"evaluation_at","evaluated_at":"2026-02-13","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"832c90e99087e0af67ac8b7df9705f9d038798c860ae16a3ddd50854fd5d0242","metric_details":{"modality_lane":"not reported","notes":null,"num_parameters":753864139008,"pull_request":null,"result_file_url":"https://huggingface.co/zai-org/GLM-5/resolve/main/.eval_results/hle.yaml","source":{"author":{"_id":"63e0eea7af523c37e5a77966","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1678663263366-63e0eea7af523c37e5a77966.jpeg","followerCount":485,"fullname":"Nathan Habib","isHf":true,"isHfAdmin":false,"isMod":false,"isPro":true,"isUserFollowing":false,"name":"SaylorTwift","type":"user"},"isExternal":false,"name":"Model Card","url":"https://huggingface.co/zai-org/GLM-5"}},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"not reported"},"run_count":null,"scaffold":null,"tools_allowed":"not reported"},"protocol_fingerprint":"e1f956386a97e6ba42a3c396757902d19f19cdabb10fcc91e345af5c771a2890","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":30.5,"score_display":"30.5","source_stated_rank":41},"run_fingerprint":"96627bdf16332ed8e053016cc6eb363688521978462c514947f91c263ac1214d","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bc699530339dcbdd472b963","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_fa9e908ac04b067eb00182d4","split_or_window":"published-leaderboard","unit":"percent","version":"HLE official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":234,"at":"2026-02-13","basis":"evaluation_at","evaluated_at":"2026-02-13","first_observed_at":"2026-08-29T23:52:43Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"830dd282e33bf176e1d8ddfdaa54027fc6b40204821403c078f1c4284cfa2e32","metric_details":{"modality_lane":"not reported","notes":"With tools","num_parameters":753864139008,"pull_request":null,"result_file_url":"https://huggingface.co/zai-org/GLM-5/resolve/main/.eval_results/hle_with_tools.yaml","source":{"author":{"_id":"63e0eea7af523c37e5a77966","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1678663263366-63e0eea7af523c37e5a77966.jpeg","followerCount":485,"fullname":"Nathan Habib","isHf":true,"isHfAdmin":false,"isMod":false,"isPro":true,"isUserFollowing":false,"name":"SaylorTwift","type":"user"},"isExternal":false,"name":"Model Card","url":"https://huggingface.co/zai-org/GLM-5"}},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["hle_judge_mismatch","hle_not_canonical_full_scale","hle_sample_count_mismatch","hle_split_mismatch"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"tools":"yes"},"run_count":null,"scaffold":null,"tools_allowed":"yes"},"protocol_fingerprint":"85de386bb5b9e508e9153a921e304c475ad7ad4f383514d2db7e317cbe2264ed","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":50.4,"score_display":"50.4","source_stated_rank":13},"run_fingerprint":"c27b37f47ca229151eb494159ded09e63685e22b9c250df7fa749e4f0e2d26a7","source":{"content_hash":"77871cb25e1d31cf82bcbded8c62ae511eafd2936054fe150e8a55b9c28a0ba7","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-05T12:33:30Z","homepage_url":"https://agi.safe.ai/","id":"hle","last_fetched_at":"2026-10-05T12:33:30Z","license":"Score metadata only; questions are gated","locator":null,"name":"Humanity's Last Exam","redistribution_policy":"score-metadata-only","result_url":"https://huggingface.co/api/datasets/cais/hle/leaderboard?limit=200"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8bc699530339dcbdd472b963","provider_config":{"source_id":"hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"C","evidence_rank":5,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"score-metadata-only","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"self-reported","verified_status":"self-reported"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"99208c05177985e288e2cb7f6ac5783cf21c12a4c0bf83528003d97717134b27","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.034091,"date_is_proxy":true,"latency_seconds":242.523,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.873},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":82.828,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":131072,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5-thinking","zero_shot_accuracy":83.838},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"6a6ffbd15be06c9d64fa3a60a22ad24ec4f0d7a211417106cf20ea085c549442","result":{"confidence_high":87.00408,"confidence_low":79.66192,"sample_count":396,"score":83.333,"score_display":"83.33","source_stated_rank":58},"run_fingerprint":"12d795d8d57d3a3b2ba11e1dbf1824618c53fbebc7f840ca16a4589863494d6d","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_c3d225c30664ec01fcf514bd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":234,"at":"2026-02-12T17:19:26.257Z","basis":"evaluation_started_at","evaluated_at":"2026-02-12T17:19:26.257Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fc8d1e9f3ad4206cbc2db54a09809d1342473f3d43fc1e3ce1ad729f3e4e4f02","metric_details":{"best_score_across_scorers":"0.8781725888324873","model_release_date":"2026-02-11","stderr":2.336331210092387},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"n4hyz75SVpwPEisKVVjvNg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fn4hyz75SVpwPEisKVVjvNg.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/n4hyz75SVpwPEisKVVjvNg.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"a5a6095421e9f04f736458d35e89a35de6e73bb10d5f411d5902e0fcef20aac1","result":{"confidence_high":92.3964680550298,"confidence_low":83.23804971146765,"sample_count":null,"score":87.81725888324873,"score_display":"87.8","source_stated_rank":68},"run_fingerprint":"f87b1f2412379d2fe7abb4d544c01b180517e34f539cb152be709090e101576d","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2d7fe4295764abffbfe59b5f","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0ebfd0edd7994a5b9a95ceae5ccef77645e3fd4fbe6248694d9a8e666b1d4695","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.011542,"date_is_proxy":true,"latency_seconds":86.889,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.343},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":null,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"zai/glm-5-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"931a11317dc231af0303e085df51ac2e1f225c46ac4b9789903b351f89797433","result":{"confidence_high":86.70228,"confidence_low":85.35772,"sample_count":null,"score":86.03,"score_display":"86.0","source_stated_rank":51},"run_fingerprint":"acf22a1edf7f9d398348c88cc915d250905b4d8d53fba8684666462ba59ed205","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_c3d225c30664ec01fcf514bd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":234,"at":"2026-02-12T16:08:22.578Z","basis":"evaluation_started_at","evaluated_at":"2026-02-12T16:08:22.578Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fd3c9fb5d469c9067e2e8fd4b606fa0a39d2b8414298bde8f443f43af0c6e842","metric_details":{"best_score_across_scorers":"0.8","model_release_date":"2026-02-11","stderr":6.0},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"ccbx7KQY9bTD563vzBXwSg","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fccbx7KQY9bTD563vzBXwSg.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/ccbx7KQY9bTD563vzBXwSg.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"24834bb9e849cfb43ac09a1b9c8e00f0dfb2ce51bb07c739eb6f0dcde7817da6","result":{"confidence_high":91.76,"confidence_low":68.24,"sample_count":45,"score":80.0,"score_display":"80.0","source_stated_rank":110},"run_fingerprint":"c7994154d4171a8603bbb421af821a62f70efc3c54aecd9920c1b8cf7587409f","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_47f623c312e058af8554fde4","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6cf97d613f9ae4cc63e4f18e4d1425ce5efcecb9acf3ba2bdbdc34559d274c92","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.27,"model_release_date":"2026-02-20T00:00:00.000Z","results_url":""},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"","model_type":"CoT","upstream_model_id":"glm-5"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a5ac13d6ba1560e9e9fe4f99107a0952e138420cd6e6f9258afc35b2f41425cf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4.859999999999999,"score_display":"5","source_stated_rank":165},"run_fingerprint":"c59e0c690e4d4d58ee20ec3bfbd5e63cc7998e0fbf2392edf68822f93e8c43ad","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_476fbefcb74061bff19d00f4","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"743e18753ab77799641ac59c4bfbea9b635e54d3e97a24aeeeb8cdee68dcdcb5","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.27","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-11","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5","source_effort":null,"source_model_version":"glm-5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"ba30d18d6c2dee850599ac493b37f84841fd80798c47c6d400f59c3172666a3d","result":{"confidence_high":7.596956,"confidence_low":3.07626,"sample_count":360,"score":4.859999999999999,"score_display":"4.9","source_stated_rank":161},"run_fingerprint":"832bbc70c43dd67e53d47b154830a3e37bdd54be1f80f7fe41f67812700edeb5","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c491273c9d347761a5e1f765","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"67d03d79b19a44189d9daf43ee5479a4c56241147f438713009ccd72315cfbc7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-11","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"ba21d91c69b2bfb5562da24595fe712c6aa16a43bea96d18b26d0459c948a095","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":53.2,"score_display":"53.2","source_stated_rank":45},"run_fingerprint":"9c2d876fbb44887b1600b9fc4759d057364b8d18732fb7ed3906e714627f2048","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_927bf51b0edb096333435e1c","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":234,"at":"2026-02-12T17:19:31.215Z","basis":"evaluation_started_at","evaluated_at":"2026-02-12T17:19:31.215Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"afc53bbb1203707c3797919eb93ffa68921ed2323ee284cdb96fddb52d629e10","metric_details":{"best_score_across_scorers":"0.1","model_release_date":"2026-02-11","stderr":3.015113445777636},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"7czodbcgEF5zN68ppPFzBN","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7czodbcgEF5zN68ppPFzBN.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/7czodbcgEF5zN68ppPFzBN.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"a1b9960561cae52d551b3addd6efc2c5013d6254a3de5b36c322981fd878f3bb","result":{"confidence_high":15.909622353724167,"confidence_low":4.090377646275833,"sample_count":100,"score":10.0,"score_display":"10.0","source_stated_rank":137},"run_fingerprint":"97961df623d7e0d5a924d81d7bc622ab85bbc4e3ff104d14cd7ec20ad4801463","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fdb4970e71e9f5d6c4a5a2d6","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"6a60974c32cccb384d8ae183156f51061505cef430eb9e950f0178a40667d85c","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.17","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-11","notes":null,"upstream_source_url":""},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5","source_effort":null,"source_model_version":"glm-5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"54207793d4d27d8f38a115533712e157798c696fa5d5368797e5c8a606bbf29a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.67,"score_display":"44.7","source_stated_rank":174},"run_fingerprint":"62cadb204d0b1675a3649bc05c35690718af78d5c8f25e95941bc5331a889005","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c6d6445cb7c06336b725fc29","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a70653ca4ddfb94a325786c84b053e11cbea6030c35a3d346cd7617b3f05a3ab","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.448911,"date_is_proxy":true,"latency_seconds":513.842,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.023},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":131072,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5-thinking"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"75433a5b378152810f23a437ae311ccb18cec772810540ed61464269d06585da","result":{"confidence_high":75.36508,"confidence_low":67.43492,"sample_count":500,"score":71.4,"score_display":"71.4","source_stated_rank":57},"run_fingerprint":"5ed4367d45d638c268e6022e062ac11b6d45656fd9956f392871589a1137bda2","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_c3d225c30664ec01fcf514bd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":230,"at":"2026-02-17","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17","status":"stale"},"measurement_id":"67de92534c9e245f7380c284f8d6bea6b77d98e6e7d0e3a7cec9f5538238120c","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_glm-5-high","logs":null,"model_display":"GLM 5","model_org":"Z-AI","reasoning_effort":"high","resolved":72.8,"site":"https://z.ai","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: glm-5","Org: Z-AI","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_glm-5-high/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"72beb0a0bc30e0472492d618ab9c630988651ae6d82335cbba25965971baabf7","result":{"confidence_high":76.515656,"confidence_low":68.736662,"sample_count":500,"score":72.8,"score_display":"72.8","source_stated_rank":17},"run_fingerprint":"e2d8dcb5b1492b81eca661f626fb9806d4baccb60505208e4421f96818292e08","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_a08d9d0d08c8cc75df725025","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":231,"at":"2026-02-15T15:40:10.848Z","basis":"evaluation_started_at","evaluated_at":"2026-02-15T15:40:10.848Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"3a078463e979b9748b3b7fe7ab01df5d21c43fba682fa5061d65bee98e66fc12","metric_details":{"best_score_across_scorers":"0.7207792207792207","model_release_date":"2026-02-11","stderr":2.0894141602415233},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"hKkJEu4ueLiNWsTtrJ78PS","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhKkJEu4ueLiNWsTtrJ78PS.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/hKkJEu4ueLiNWsTtrJ78PS.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"5d983b02c02935e00203af862de22e049304dcd2f74d9dbb0233ab54411812ba","result":{"confidence_high":76.17317383199546,"confidence_low":67.98267032384868,"sample_count":484,"score":72.07792207792207,"score_display":"72.1","source_stated_rank":23},"run_fingerprint":"77a37394c1dea6460c0a71e65e1b352c5b3cb24b77b8daec149513fbce90734b","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cea0f98a1c5f5f2a3ae0c8af","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_7449132b5f078c71f23c26b5","split_or_window":"2026-02-15/2026-03-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":218,"at":"2026-03-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"39af4b113159dd99644b030d734259a14ba4ec78dbafef15442807f608da7a95","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-02-11","pass_at_5":70.37037037037037,"potential_contamination":false,"sem":1.1712139482105108},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"GLM-5__tools","window_from":"2026-02-15","window_status":"historical-post-release","window_to":"2026-03-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"3a19811f492e0107b51dc9de4ab7d657eafc63c27b4c368713b29690d8fd2d14","result":{"confidence_high":65.25854230145556,"confidence_low":60.66738362447036,"sample_count":null,"score":62.96296296296296,"score_display":"62.96","source_stated_rank":null},"run_fingerprint":"7d6cdf91cf00098d20807b8d5fd993c19ce5780a669f1f5e519dc4c0adea7078","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d4385bff71d0ba7da64cdc30","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b6bfefeb1bf8c2afbbd9d5425e9a74e75e838704551d70a8be2fa9c9e3fa41bd","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.043918,"date_is_proxy":true,"latency_seconds":289.752,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.057},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":null,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"zai/glm-5-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"71cba4553c5ed1c67f1df4a384d018b46e86356b12b2160919c2741f5dd02d86","result":{"confidence_high":83.93972,"confidence_low":79.79628,"sample_count":null,"score":81.868,"score_display":"81.9","source_stated_rank":59},"run_fingerprint":"b1f73f4161c1db8b857d753c7256997babf0b37e37886cb6563ba7402a2c305f","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_c3d225c30664ec01fcf514bd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b2a17685743d36fe8c3f12eb8c52cc7709c473f7c62b97417b261c9e8da89310","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.073284","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-11","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"glm-5","source_effort":null,"source_model_version":"glm-5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"1e275173e9df9ea2c30f14c0e4badff4ff8640ece726084771e1affe20ef4804","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":765.62,"score_display":"765.62","source_stated_rank":72},"run_fingerprint":"780446d9ac273762b26c93fff1874f6b1e86a57b3173d21086d3d661d8e0e69e","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a6a581cf3925bd5dfbd5710f","provider_config":{"source_id":"epoch-ale-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d224a63ff3808764190749ae095b807968c033eb77d74cb99209b693e7c43949","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":40.270258,"date_is_proxy":true,"latency_seconds":13455.794,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":4.034},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":131072,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":0.95,"upstream_model_id":"zai/glm-5-thinking"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"90308894920d0819d9ca6be09583c8487224f33e5e78c9de834deb1c16b08194","result":{"confidence_high":31.26564,"confidence_low":15.452360000000002,"sample_count":null,"score":23.359,"score_display":"23.4","source_stated_rank":71},"run_fingerprint":"2f094f9c24e05396a4e8ac11eb1d242368149141ecce3edd9f9d28c645721d18","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_c3d225c30664ec01fcf514bd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8aec51d8e77ffff3db30cb5fbe09f90e086822dd50f1bb424aa94b25d3af1a87","metric_details":{"license":"MIT","variance":17.19319608354095},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"glm-5","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4f8d71031197c0244b3a929427b58cc6db620dcc743c9bbb634087d777c4f2fb","result":{"confidence_high":1442.1679406595938,"confidence_low":1425.9140851476186,"sample_count":7504,"score":1434.041012903606,"score_display":"1434","source_stated_rank":76},"run_fingerprint":"e220bc05c2478527dd9f7a05312a5fe3c73255af31162940d2e1237ae0e73a37","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fcf17c516dbc641b9250907f","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b4bf793dcd3bfbdb879bb89922edaf312f80bda88531bf5ee7ec814966a5c307","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-11","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"4f440553700d971a5bfd66c08b7fc9b61741ed67afba77512052d88a69431b46","result":{"confidence_high":1444.25,"confidence_low":1427.56,"sample_count":7056,"score":1435.91,"score_display":"1435.91","source_stated_rank":74},"run_fingerprint":"ba9f3ea8a058e00cb359c836a9e79a99010dd7a9319c8321c980283325a4a807","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6c288c5ace23ef8c03125cca","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5f441dec12857735e04c119865e10be548cf78b3b9fac4a79483d747413d8055","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-11","notes":null,"upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"glm-5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"4c2a0f437452a47939588543ed5de59f8baa840b89d7cdfba721482c4cfae020","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":48.17,"score_display":"48.2","source_stated_rank":75},"run_fingerprint":"f8d46d3711bbaee197bcba4c656174a8f5669413636e8ed456723e572ea263ab","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_818aa5ca47a72c9303b2a24e","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":236,"at":"2026-02-11","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:16Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-11","status":"stale"},"measurement_id":"f834efc0c4b1e8af1def1b695cdadf7eecff70dc6be00abddc91ebd219520676","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-11","notes":null,"standard_error_points":2.6265894906,"upstream_source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0"},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"GLM 5","source_effort":null,"source_model_version":"glm-5","source_row_date":"2026-02-11","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"c45fd3f81adfce35d09494885f73c638300e7c8d57411e768c137e31b057e63c","result":{"confidence_high":57.563845738675994,"confidence_low":47.267614935523994,"sample_count":89,"score":52.415730337099994,"score_display":"52.4","source_stated_rank":67},"run_fingerprint":"5b612038af07a3d38354d3f0807d99536192d8c6e46b1f469b9a645a68e09c7c","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f1c0dae42e282c253a7249e5","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":220,"at":"2026-02-27","basis":"evaluation_at","evaluated_at":"2026-02-27","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"e12e502e514bfe0e416a3798ede9958094b912521bd5cb9aff6b662075d37d45","metric_details":{"cost":null,"pass_2":7.22,"pass_3":5.67,"pass_4":3.09},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"text-emb-3-large","submission_slug":"glm-5-think_sierra_2026-03-02","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"d56ddeb73571ba07e4be83089b0896e0f20a4d20ab1108fa0607d3706ea2a01c","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":9.79,"score_display":"9.8","source_stated_rank":null},"run_fingerprint":"728a2a4810f9681090a20430f7cda5a497bc80424d69b6fc5cc8b415a12c3a46","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ce13d8bd3f6566efc90ebe09","provider_config":{"source_id":"tau3-bench","source_label":"enabled"},"provider_mode_key":"enabled","raw_label":"enabled","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"17489090dab36eadce4b9ee5f1e076ef6f5e6efc6582e42e53f41273e74a79bd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.308","mean_standard_error":"2.8","model_release_date":"2026-02-11","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.7,"upstream_source_url":null},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM 5 (Thinking)","source_effort":null,"source_model_version":"glm-5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"66815f13dabe6e99041b2b91074dc97d9ba83205878237747f33b7ac154482c8","result":{"confidence_high":22.491999999999997,"confidence_low":11.908,"sample_count":480,"score":17.2,"score_display":"17.2","source_stated_rank":43},"run_fingerprint":"004eaa93e83d9ef0ecd5bf6652ca3d8fb9bc5d72bc6b15f0529afaf03fb66ea1","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_c3d225c30664ec01fcf514bd","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9cde23ffeb0586fd7e5167cbda02c9ed9282d4755444ffc797c144eee883c661","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-02-11","notes":null,"upstream_source_url":null},"model":{"id":"zai/glm-5","name":"GLM 5","provider":"Z.ai","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GLM-5","source_effort":null,"source_model_version":"glm-5","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"554f9877bfc4e217d97cca49b51d650a61d333d955822e66a234c870093a445d","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4432.122000000001,"score_display":"4432.12","source_stated_rank":34},"run_fingerprint":"1db90918d376210a90882cc4e89c3b93c921ff2d1ce0ecf734a351742aa605f2","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cf429bf4453c02a3a83fad37","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"8d68a71afdd49ed711e974334af4f7d12fd9e0269a915da522c0e5aa71e9c1e4","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-09-29"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Sonnet 4.5; model release: 2025-09-29","source_accessibility":"API access","source_model_name":"Claude Sonnet 4.5","source_model_release_date":"2025-09-29","source_model_versions":["claude-sonnet-4-5-20250929","claude-sonnet-4-5-20250929_12K","claude-sonnet-4-5-20250929_16K","claude-sonnet-4-5-20250929_2K","claude-sonnet-4-5-20250929_32K","claude-sonnet-4-5-20250929_48K","claude-sonnet-4-5-20250929_59K","claude-sonnet-4-5-20250929_unknown"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"1b1565eb7ab3a87fc31a6a2e0376ceb564d463d6f71af067c4f3b4a8039dec07","result":{"confidence_high":148.51,"confidence_low":145.34,"sample_count":null,"score":146.84,"score_display":"146.84","source_stated_rank":66},"run_fingerprint":"e271bf7f0fa8df0352fffd28191a6706d5486aefcec0d811b9cf890dc43c46a6","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d4c6e9282aafa0f6c5e6104b","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"20a6802c902b8ebb38625f32be752a5d6bf72f7e1f6483410a14b552c800080a","metric_details":{"license":"Proprietary","variance":1.994452177390197},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1459.3640764820386,"confidence_low":1453.8281552725962,"sample_count":83670,"score":1456.5961158773173,"score_display":"1457","source_stated_rank":65},"run_fingerprint":"b143dea72edbbaa523cf7ed84ca87b492b230bb4346b479eba7eb3e0513c60af","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cf2150debee34b4d5f1bf4b9","provider_config":{"source_label":"high","token_budget":32000},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"60cac3b572bbf0e0a39de1e7c2ed76eb40eb700c0ac4f961149e7a65d176ba42","metric_details":{"license":"Proprietary","variance":2.1414043703965358},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1458.038515197359,"confidence_low":1452.3022736726905,"sample_count":82425,"score":1455.1703944350247,"score_display":"1455","source_stated_rank":70},"run_fingerprint":"29ac35c85e9a2825103fc8553e94d7e21f1cd5cb29b7584cdaba9dd72ef96d46","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1ba24c559eaf3ba6e18ee0c7","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_7bf989f19fbe70ed22a4a50b","split_or_window":"verified-1000-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":55,"at":"2026-08-10T22:42:10.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-10T22:42:10.000Z","first_observed_at":"2026-08-28T22:06:10Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d5170f7df2eaf36f6aa72b857d0b413dcbd7497bdf74b1b6d29c16bf5830c5e6","metric_details":{"best_score_across_scorers":"0.237","model_release_date":"2025-09-29","stderr":1.3454070462577934},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"MhqeuRVx4JcRfxPQDYPq7Z","log_viewer":"","logs":"","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"17365a273d377061f9af1db8b6ceea460b5ef524be9d11495cc86c480c6dae13","result":{"confidence_high":26.336997810665274,"confidence_low":21.063002189334725,"sample_count":1000,"score":23.7,"score_display":"23.7","source_stated_rank":70},"run_fingerprint":"9bb16b84b1bb0757be644dbd4a71bd8eaf75f5f9cf3529ba1a3d4f02c600a91e","source":{"content_hash":"6c9947bda3dafe8e22f2662a05d9c5732ef430bc28049ccc2e8eeb713555d61e","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-01T20:29:30Z","homepage_url":"https://epoch.ai/benchmarks/simpleqa-verified","id":"epoch-simpleqa-verified","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simpleqa-verified"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1ce9cf4baebaf695f5b05969","provider_config":{"source_id":"epoch-simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":323,"at":"2025-11-16T00:00:00Z","basis":"evaluation_at","evaluated_at":"2025-11-16T00:00:00Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"20685cde7c71642826b9c1ed4711ef10f468227417a6c768d1859da8ed386e3d","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-sonnet-4-5-thinking-20250929","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"7e40e7298e5ea5551ab9601e90dd3d533af321510c431e202bce60f78b03f620","result":{"confidence_high":32.37752924265366,"confidence_low":26.0463838008246,"sample_count":null,"score":29.21195652173913,"score_display":"29.2","source_stated_rank":26},"run_fingerprint":"f70a24b2541c555e0e735260f18a7707163b07b79e71a4132202a5c90e8d16c2","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_642e3a37222b1abf74918908","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":323,"at":"2025-11-16T00:00:00Z","basis":"evaluation_at","evaluated_at":"2025-11-16T00:00:00Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"49bdede58245dd2ba2fc60fd1ed62016dad3c9cda54ad84f778a4f98ca5777b1","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"claude-sonnet-4-5-20250929","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"d62208085ceba56d71c319217754adc1ca422d42d422d0de7b767571b945b37c","result":{"confidence_high":22.487727805858952,"confidence_low":16.14525882118413,"sample_count":null,"score":19.31649331352154,"score_display":"19.3","source_stated_rank":34},"run_fingerprint":"48372f14f4d5c85d0bee3a209dcd1ce6e3ce9e91258334cb9b00da0c045e6439","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_717737b710baf06c1d2930e0","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2fad9b7a8af3b353619c8626614ea3331b77a3d8001a89e0bacb2df37d65449a","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.114262,"date_is_proxy":true,"latency_seconds":117.251,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.254},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":81.633,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":64000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-5-20250929-thinking","zero_shot_accuracy":81.633},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"3f405f74303500a46d9002d0b1e4ac6b9cbe3dabef77e4efba869d0f8cd0f511","result":{"confidence_high":86.05084,"confidence_low":77.21516,"sample_count":396,"score":81.633,"score_display":"81.63","source_stated_rank":63},"run_fingerprint":"888e48099e8950a14277cd7116320a9552bcd0136da8a6e9fcd380ddd5190802","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_642e3a37222b1abf74918908","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e0d5a8fd77c230420b825df7c324c74e2d2694a461318eca7206e0108e957a85","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.040152,"date_is_proxy":true,"latency_seconds":38.2,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.391},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":64000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-5-20250929-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"8626b6b76fb3ab8455f50505c3dcf89d51b99d1a871b7361056c58a3ab68873d","result":{"confidence_high":88.12336,"confidence_low":86.59064,"sample_count":null,"score":87.357,"score_display":"87.4","source_stated_rank":28},"run_fingerprint":"43dad3a0460b91d2d50ab87a78000fa34f0dcf965ef51a48323d7d2a9840146f","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_642e3a37222b1abf74918908","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T21:10:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T21:10:25.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1fe2145d5b3b1a48c9570dad6c8c0325e8c8f11bcc52988e4fef6130981f22b8","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.23859649122807017","leaderboard_private_problem_count":285,"model_release_date":"2025-09-29","public_example_count":10,"stderr":2.5291832284114157},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["32k thinking"],"epoch_id":"7oUkeLfwMK3MtkSNHX6QPu","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7oUkeLfwMK3MtkSNHX6QPu.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/7oUkeLfwMK3MtkSNHX6QPu.eval","manual_evaluation":"","reasoning_token_budget":32000,"service_tier":null,"source_model_display_name":"Claude Sonnet 4.5 (32k thinking)","thinking":{"budget_tokens":32000,"type":"token_budget"},"upstream_model_id":"claude-sonnet-4-5-20250929_32K","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"2ef49575354bf730e05acc6ad0cd5064457f40531f1dbdd9ad25e49f1ceedef2","result":{"confidence_high":28.816848250493393,"confidence_low":18.90244999512064,"sample_count":285,"score":23.859649122807017,"score_display":"23.9","source_stated_rank":79},"run_fingerprint":"797eb44d87107400c273ace380de9775a60eef869fa0c1c021ba48194f2cbdb6","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_34919f24dc735cc9bea725dd","provider_config":{"source_id":"epoch-frontiermath","source_label":"thinking-32k","token_budget":32000},"provider_mode_key":"thinking-32k","raw_label":"thinking-32k","reasoning_enabled":null,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2d826969eb437b0de3f0971bdb85024784e7654e6b9c304883f954e333c3bd5a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.7589,"model_release_date":"2025-09-29T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-sonnet-4-5-20250929","model_type":"CoT","upstream_model_id":"claude-sonnet-4-5-20250929-thinking-32k"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6b9c2fd7654c4f0f9bfb026fcfc72a0b96661576340141c91daef92c4047280f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.61,"score_display":"14","source_stated_rank":137},"run_fingerprint":"34a0a6024ffc9b5e26cfd52d72533591d6049d192064fd3375f918f5e6fddaf9","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_beda1f27c08006aeb0f60b10","provider_config":{"source_id":"arc-agi-2","source_label":"Thinking 32K","token_budget":32000},"provider_mode_key":"Thinking 32K","raw_label":"Thinking 32K","reasoning_enabled":null,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ea307563f684e4c7fafc99c99ef5bf73ca610aaf73a67b5b697810b83d7893b3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.3499,"model_release_date":"2025-09-29T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-sonnet-4-5-20250929","model_type":"CoT","upstream_model_id":"claude-sonnet-4-5-20250929-thinking-16k"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e9ae98b7e0c2594e36042aa0b3c0f518f215ab10c43cde5cbec8b1d72ad78ee8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":6.94,"score_display":"7","source_stated_rank":152},"run_fingerprint":"0cc5d6fbf8f66945bd119b67fd599245b8ef427c657d23d292f67bd889d1f389","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_70ba794ba779e4e1b4fd260d","provider_config":{"source_id":"arc-agi-2","source_label":"Thinking 16K","token_budget":16000},"provider_mode_key":"Thinking 16K","raw_label":"Thinking 16K","reasoning_enabled":null,"strategy":"token_budget","token_budget":16000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ec6e45244fe500c5bc90aa3ce2b74d64d8c4bef33a6388fdf622c60b0f6717be","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.2349,"model_release_date":"2025-09-29T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-sonnet-4-5-20250929","model_type":"CoT","upstream_model_id":"claude-sonnet-4-5-20250929-thinking-8k"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"ed35928907a133d88181e4a365120296960ecc7a7222cdbb26762ddd9c65ed87","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":6.94,"score_display":"7","source_stated_rank":152},"run_fingerprint":"2aad365cadd8ab06bb4b6242f427d90c165e0c62ce64cfc16d973aa94fcc794a","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fd739f70c64b5804842107a6","provider_config":{"source_id":"arc-agi-2","source_label":"Thinking 8K","token_budget":8000},"provider_mode_key":"Thinking 8K","raw_label":"Thinking 8K","reasoning_enabled":null,"strategy":"token_budget","token_budget":8000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dc45ce4921f6bd977f8ace25982f57d8a47791d623acb557afa35b8be97055a1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.1424,"model_release_date":"2025-09-29T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-sonnet-4-5-20250929","model_type":"CoT","upstream_model_id":"claude-sonnet-4-5-20250929-thinking-1k"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f8731bd5f5f8b0bcb8ef0f32762e9c141bae11b88db8a25f50a75de43d94a322","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":5.83,"score_display":"6","source_stated_rank":159},"run_fingerprint":"423c31bf965f23e72041d7aaa597ef59b621fb94ece9c4bcd55ef6e05a4373d3","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a7936642f092bb163ee3b076","provider_config":{"source_id":"arc-agi-2","source_label":"Thinking 1K","token_budget":1000},"provider_mode_key":"Thinking 1K","raw_label":"Thinking 1K","reasoning_enabled":null,"strategy":"token_budget","token_budget":1000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4e959cad5720f7d8e81841902e0f31aa840ed090834c3fbb61861b0ef0ef5c86","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.1295,"model_release_date":"2025-09-29T00:00:00.000Z","results_url":""},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"claude-sonnet-4-5-20250929","model_type":"Base LLM","upstream_model_id":"claude-sonnet-4-5-20250929"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"69bff0e3242d192105c013fbc1b0a8b5f0ff0781b497f37e06715e1c3866bdab","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3.75,"score_display":"4","source_stated_rank":178},"run_fingerprint":"e809f9697c147c59ced113ce624c45e7e11cf7f2755fb4cc1621f52c2f498481","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c7d823058da1444a086b741f","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"2b77d30eaff04aefa514f93e491edd4d04c7e4c0ebfe827d2be3e3b1c9a06d4b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"0.1295","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":6,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.5","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"23679344db135975e66d9df2f43309a68de8c78a4b28f7552945f779d4ebd508","result":{"confidence_high":6.25064,"confidence_low":2.226015,"sample_count":360,"score":3.75,"score_display":"3.8","source_stated_rank":174},"run_fingerprint":"834afa59bf1af38b94e738845843a8689fbaa22859405d6845dba6daea3e315a","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_48e30e06d9bd892cc6356a95","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T21:10:25.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T21:10:25.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"18fa9b2ff10815a3cbccc42f5fd6f2c67597b668bc0f9120911f442c098f7209","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.024390243902439025","leaderboard_private_problem_count":41,"model_release_date":"2025-09-29","public_example_count":2,"stderr":2.4390243902439024},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["32k thinking"],"epoch_id":"d3m9Xip8cof4s77z6LKkEh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fd3m9Xip8cof4s77z6LKkEh.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/d3m9Xip8cof4s77z6LKkEh.eval","manual_evaluation":"","reasoning_token_budget":32000,"service_tier":null,"source_model_display_name":"Claude Sonnet 4.5 (32k thinking)","thinking":{"budget_tokens":32000,"type":"token_budget"},"upstream_model_id":"claude-sonnet-4-5-20250929_32K","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"b012c400b53926d881db508924e56cc34a6750fad8cb9a2c32cefa8a1c6bebd3","result":{"confidence_high":7.2195121951219505,"confidence_low":0.0,"sample_count":41,"score":2.4390243902439024,"score_display":"2.4","source_stated_rank":61},"run_fingerprint":"f969d28c1f0c102a2f047ef46787e06a1065854bdbe9ab354bfa5635601ed30b","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1749eab61af56fe6c62ab77a","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"thinking-32k","token_budget":32000},"provider_mode_key":"thinking-32k","raw_label":"thinking-32k","reasoning_enabled":null,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8e9561173b4edaa7db39298dcba25f1e7689b89ffdb80709a93892d586f8af8a","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.058232,"date_is_proxy":true,"latency_seconds":56.361,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.974},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":64000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-5-20250929-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"5ad6880b5a00668161c139cba1940d055a3f64178ab634743fda403e2c2ca7f6","result":{"confidence_high":81.21504,"confidence_low":77.39695999999999,"sample_count":null,"score":79.306,"score_display":"79.3","source_stated_rank":48},"run_fingerprint":"4d7492f7c2ecfc98d0fd17c72dc663602dc8ba7b8cda909f6a1dfbc4b84f853b","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_642e3a37222b1abf74918908","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ca6110cb71040915b63770b061c75a043f861848f8d464d089ccf4bd693e14a3","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.5 Sonnet (Reasoning)","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"0dc06ea062155b9a4ba5cbf0d3ea98569889dd8b4f016dcf20079beca3db48cb","result":{"confidence_high":7.127519,"confidence_low":0.173844,"sample_count":71,"score":1.1428571428571401,"score_display":"1.1","source_stated_rank":140},"run_fingerprint":"e4cefd99ca6599ceb7456060a9db7d72821ba882518bcec51665ecd30a6b2470","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_3faedfeaf2b018b3d825930d","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"92f2e9b59f8f4545c70a6dd42be21fe61b5f943d612c28dd8df46cea7d1ba89f","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"5cd245fe69b46f91f733fb3a26f963748494b2b134260f93e466648553703f57","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":54.300000000000004,"score_display":"54.3","source_stated_rank":43},"run_fingerprint":"8dadf6066e9a417117e2e8f570f61be9077ce62f8d6be392c3819fee70948897","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8c431ef1c3357f4933de067d","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:21:13.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:21:13.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"6e8425c8bd8a42012006e5ce1263c76365c0ce4bdffe3834a3a74d764263b086","metric_details":{"best_score_across_scorers":"0.04","model_release_date":"2025-09-29","stderr":1.969463855669324},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"KB62zrQQ5ccd7hCcUwkX8j","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKB62zrQQ5ccd7hCcUwkX8j.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/KB62zrQQ5ccd7hCcUwkX8j.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"b724314a22a787c1e2a4135a5758fae05e1a931cdd926f82b122fa2cfe8d831c","result":{"confidence_high":7.860149157111875,"confidence_low":0.1398508428881251,"sample_count":100,"score":4.0,"score_display":"4.0","source_stated_rank":169},"run_fingerprint":"11d0efc2d430a975b3a5ee457ddb9acd7446377fbf8f6e14696fd54c2d724e79","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6a469e49a01c32309c33941b","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e8cd0a182de971b6e15735433a27de1e799b88411adc0a3ab6ce5b697ee04502","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.0808","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":""},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.5","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"927a7a84c5b9eaaebb3c9c04a4e8184369edc2fa0aa7bae89f816e11c3f637c4","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.5,"score_display":"25.5","source_stated_rank":217},"run_fingerprint":"19b4180b310df6b1978655b252659a1245273d2c4faf09201049ac5875a207e6","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0de451a2a2c91a817f1dd3ec","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e3a0d4eb130fd66d502a771fd55ead8f7681c12a8bf0d6d9c835f096a45ec5bc","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.807156,"date_is_proxy":true,"latency_seconds":360.558,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.051},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":64000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-5-20250929-thinking"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"6cc70255689c34c61d0075a67204a97cc5e3172ac4c6424851c7e0f7f6179ce2","result":{"confidence_high":74.01996,"confidence_low":65.98004,"sample_count":500,"score":70.0,"score_display":"70.0","source_stated_rank":61},"run_fingerprint":"85d07305e8c19d5a7732508f93784c47f5fa8829e2db0a5e15081f060e42590e","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_642e3a37222b1abf74918908","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":230,"at":"2026-02-17","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-02-17","status":"stale"},"measurement_id":"9f8f46dcd4641625ec67f35521334a71d8561abc1990153daf6eadb171c04a09","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"mini-SWE-agent","checked":null,"date":"2026-02-17","folder":"20260217_mini-v2.0.0_claude-4-5-sonnet-high","logs":null,"model_display":"Claude 4.5 Sonnet","model_org":"Anthropic","reasoning_effort":"high","resolved":71.4,"site":"https://www.anthropic.com/claude","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: claude-sonnet-4-5-20250929","Org: Anthropic","System: Attempts - 1","Mini: 2.0.0"],"trajs":"s3://swe-bench-submissions/bash-only/20260217_mini-v2.0.0_claude-4-5-sonnet-high/trajs"},"run_count":null,"scaffold":"mini-SWE-agent","tools_allowed":null},"protocol_fingerprint":"7df6ebab7f439b0df9d75a613266f61bb4344f2b620139c3e2202ea20ca613c9","result":{"confidence_high":75.186061,"confidence_low":67.287605,"sample_count":500,"score":71.4,"score_display":"71.4","source_stated_rank":23},"run_fingerprint":"6293e926b20d9667ed88dd59636ac3e5b3798dce5e0aa65d385f35c4246281bc","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_7472b866192341b06629128a","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":241,"at":"2026-02-05T16:52:49.622Z","basis":"evaluation_started_at","evaluated_at":"2026-02-05T16:52:49.622Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fc6e6edd30da929838be1de94faa635a7e44d5dddcb6139e27869f9dada3f5fa","metric_details":{"best_score_across_scorers":"0.7128099173553719","model_release_date":"2025-09-29","stderr":2.0587234937983245},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"Lti64yMEsBWSzgEMFCSVhp","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLti64yMEsBWSzgEMFCSVhp.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/Lti64yMEsBWSzgEMFCSVhp.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"18037ba5d3b2ec0a44cea9271fe28abee40cd3742c7e01cfe8646d44ecf18058","result":{"confidence_high":75.3160897833819,"confidence_low":67.24589368769247,"sample_count":484,"score":71.28099173553719,"score_display":"71.3","source_stated_rank":25},"run_fingerprint":"efb034d42c6d78771ab8426cb337293ee682e2f0b6bd06de87ac052f85eab2fb","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_44d46880175e756f63b43899","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_f8f1a1bcc3ff0ccb4fe9755a","split_or_window":"verified-500-official-submissions","unit":"percent","version":"SWE-bench Verified official leaderboard"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":336,"at":"2025-11-03","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-01T07:03:20Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-03","status":"stale"},"measurement_id":"e5a97bf069daa574347562ad9133286432419b73cf859768810ee604430897ff","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 500 scored items with a 95 % Wilson interval."},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"Sonar Foundation Agent","checked":false,"date":"2025-11-03","folder":"20251103_sonar-foundation-agent_claude-sonnet-4-5","logs":"s3://swe-bench-submissions/verified/20251103_sonar-foundation-agent_claude-sonnet-4-5/logs","model_display":"Claude 4.5 Sonnet","model_org":"Anthropic","reasoning_effort":null,"resolved":74.8,"site":"https://www.sonarsource.com","source_git_blob_sha1":"1609701d2d73b0f785d28d247155886062d942f1","source_revision":"193160a463a435d05cf44a1fa9dc5eac832113c3","tags":["Model: claude-sonnet-4-5","Org: Sonar","System: Attempts - 1"],"trajs":"s3://swe-bench-submissions/verified/20251103_sonar-foundation-agent_claude-sonnet-4-5/trajs"},"run_count":null,"scaffold":"Sonar Foundation Agent","tools_allowed":null},"protocol_fingerprint":"96a84b4a183d460c4b7eb8fcf2e2b3c51e8eec06d53557f5b85de6ff1a2f1355","result":{"confidence_high":78.406678,"confidence_low":70.815141,"sample_count":500,"score":74.8,"score_display":"74.8","source_stated_rank":10},"run_fingerprint":"9767cf0a134e7246c0b16e5147b85912717cc4dcf5a1d902e16cf01ff5abb797","source":{"content_hash":"83cd949a9582f4dd68b0a07148cd86ff0eae6a05b0f2d2298a8f3717baf89d2d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-01T07:03:20Z","homepage_url":"https://www.swebench.com/","id":"official-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Repository CC-BY-NC-4.0; aggregate leaderboard-score republication permitted with attribution for this non-commercial site (2026-09-03)","locator":null,"name":"SWE-bench Official · Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/193160a463a435d05cf44a1fa9dc5eac832113c3/data/leaderboards.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_81d80ac1addb9b26d5ed240d","provider_config":{"source_id":"official-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_f0409f3e3e823115a9e78774","split_or_window":"2025-10-01/2026-03-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":218,"at":"2026-03-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"fb2797a42649e5118de073994352126ab96b85c093ab401f0fee5d5e675abb9f","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-09-29","pass_at_5":66.4,"potential_contamination":false,"sem":0.8485281374238577},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"Claude Sonnet 4.5__tools","window_from":"2025-10-01","window_status":"historical-post-release","window_to":"2026-03-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"afa1c5db8191e3968fbe9572001c1e06f3535024e3785db30a617ee2b341f91b","result":{"confidence_high":55.66311514935076,"confidence_low":52.33688485064924,"sample_count":null,"score":54.0,"score_display":"54.00","source_stated_rank":null},"run_fingerprint":"f80df3436e6ca59bfa5b94d001f53be83c97dba213069c91e7e77a983d9e66e5","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8a4d978fd009422ee97e7041","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-live","metric":"resolved_rate","name":"SWE-bench-Live · Lite","release_id":"release_9d2de9d5be7e9822af19f7b3","split_or_window":"lite-300-python","unit":"percent","version":"SWE-bench-Live Lite"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":98,"at":"2026-06-29","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:08:10Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-29","status":"stale"},"measurement_id":"acecf6c8845b28d2dd091584f9240a513747f8658af3d888aa3db7f0750863e9","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 300 scored items with a 95 % Wilson interval.","leaderboard_script_url":"https://swe-bench-live.github.io/leaderboard.js","publication_status":"cleared-with-attribution","resolved_instances":36,"submission_url":"https://github.com/SWE-bench-Live/submission/tree/main/submissions/lite/sapient-slingshot-agent/v2.6.0/20260629-claude-4.5"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"Slingshot-v2.6.0","report_file":"reports-0605.jsonl","report_set":"lite","source_model_label":"Claude-4.5-Sonnet"},"run_count":null,"scaffold":"Slingshot-v2.6.0","tools_allowed":"agent-specific coding tools in the official SWE-bench-Live harness"},"protocol_fingerprint":"d33b6160d8c4f5486c61d6719087c8fc1608c55248bf81a1934a392463fb972b","result":{"confidence_high":16.165867,"confidence_low":8.795034,"sample_count":300,"score":12.0,"score_display":"12.0","source_stated_rank":null},"run_fingerprint":"7c16f58a84247605792f7607be154d4f049dd426a6e72123a0e2931dad11629b","source":{"content_hash":"e0226b6333c547885c15c6d53074d0ad5e2a7b99db2d6b54640d7d6537c242b4","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-09-30T18:53:18Z","homepage_url":"https://swe-bench-live.github.io/","id":"swe-bench-live","last_fetched_at":"2026-10-05T12:33:28Z","license":"Benchmark and harness MIT; aggregate leaderboard results redistributable with attribution (maintainers' confirmation 2026-09-02)","locator":null,"name":"SWE-bench-Live · Lite","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-bench-live.github.io/reports-0605.jsonl"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4d43287ed026bd0acbdb0e15","provider_config":{"source_id":"swe-bench-live","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-live","metric":"resolved_rate","name":"SWE-bench-Live · Lite","release_id":"release_9d2de9d5be7e9822af19f7b3","split_or_window":"lite-300-python","unit":"percent","version":"SWE-bench-Live Lite"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":308,"at":"2025-12-01","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:59Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-01","status":"stale"},"measurement_id":"b626a81762c114e585b57ca539f80a534cbec63536713ee61cd5c0a1d9da2eaf","metric_details":{"confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 300 scored items with a 95 % Wilson interval.","leaderboard_script_url":"https://swe-bench-live.github.io/leaderboard.js","publication_status":"cleared-with-attribution","resolved_instances":108,"submission_url":null},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"SWE-agent","report_file":"reports-0605.jsonl","report_set":"lite","source_model_label":"Claude-4.5-Sonnet"},"run_count":null,"scaffold":"SWE-agent","tools_allowed":"agent-specific coding tools in the official SWE-bench-Live harness"},"protocol_fingerprint":"4d816bcdd5ae496d8493d64fa41a123211099f2d2658df3cc01f86f24edebb0f","result":{"confidence_high":41.577174,"confidence_low":30.776842,"sample_count":300,"score":36.0,"score_display":"36.0","source_stated_rank":null},"run_fingerprint":"51ec70ccdd6a6b7ee97e5b9963e129014a0c6e126be23f60a941db83c37c4918","source":{"content_hash":"e0226b6333c547885c15c6d53074d0ad5e2a7b99db2d6b54640d7d6537c242b4","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-09-30T18:53:18Z","homepage_url":"https://swe-bench-live.github.io/","id":"swe-bench-live","last_fetched_at":"2026-10-05T12:33:28Z","license":"Benchmark and harness MIT; aggregate leaderboard results redistributable with attribution (maintainers' confirmation 2026-09-02)","locator":null,"name":"SWE-bench-Live · Lite","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-bench-live.github.io/reports-0605.jsonl"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4d43287ed026bd0acbdb0e15","provider_config":{"source_id":"swe-bench-live","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:04Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"93bb1f451ec6c395676c20fcf930cee39e990069dbdd7112dd40676f6a59fe5d","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":"GSO Leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":null,"source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"d0523de259379b25230c65ac521c8505beb7b2d3c498b5f68d64e26dccb9991e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":14.7,"score_display":"14.7","source_stated_rank":17},"run_fingerprint":"e55b69eea3149f66fcd549e8659951346e09fdd0c731b6d018366a92339c6339","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6590a3699935a47fe2f51b4c","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":360,"at":"2025-10-10","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-10-10","status":"stale"},"measurement_id":"1d535eeab73795b7bfb36d948a1389bd4d9e9cf7936f425c91a2b190320a0dc2","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":"GSO Leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929_unknown","source_row_date":"2025-10-10","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"1053156da4b9570c8758650091de48b6aa11deb535140fb289b3c9de94954a03","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":14.71,"score_display":"14.7","source_stated_rank":16},"run_fingerprint":"559c66bb1efece895c92e85b3083d6bb09cff8809990523961543ac0ef021df6","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6590a3699935a47fe2f51b4c","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b0817464f874e0c6a654b07a76e1cc8812ebe9b3402d80dd7e4c5e586ec98651","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.135104,"date_is_proxy":true,"latency_seconds":109.547,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.125},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":64000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-5-20250929-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"0dd71b1ac55ad1b3817da4449121a3b7e6627d5c7e5beaa9647d1942dde65928","result":{"confidence_high":75.201,"confidence_low":70.791,"sample_count":null,"score":72.996,"score_display":"73.0","source_stated_rank":83},"run_fingerprint":"67d223b2b7a30c2e19ee2a1ec858c98756dce0d1bed36b77829494b7f9c3a985","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_642e3a37222b1abf74918908","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c4be024dc9cbefa78fd0e96fc871868fced8afd301e32b408dddab051e31be28","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude 4.5 Sonnet (Reasoning)","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"17dab62fbce1564a83a9656d5ed96f4dab04756e23fd6b8fd62c27bb13d51030","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":44.6759259259259,"score_display":"44.7","source_stated_rank":126},"run_fingerprint":"5f2c4b3e0ac62d5a198851814aed95dbf125cdfd24abcbb711161055d8d50901","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_6fe722919cfa837c93d9a4c3","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8ecf3e9e205f902611c598da247ccfddb2fed40f3a7c29bd754e24eb2afabaf5","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":6.656984,"date_is_proxy":true,"latency_seconds":2962.133,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":3.728},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":64000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-sonnet-4-5-20250929-thinking"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"7a399f9ea96ffd1e1716e247d6564c3c2b1aeca19fcddfc4034e3d2c501c6795","result":{"confidence_high":29.92788,"confidence_low":15.314119999999999,"sample_count":null,"score":22.621,"score_display":"22.6","source_stated_rank":72},"run_fingerprint":"99c3be4bc91d8f591a0bd2bc0d501d5e45d38e8e78f033291c1849505820cf90","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_642e3a37222b1abf74918908","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e61c4b81d3f85800f7361eb2729b88834ad98c46437cb2942a034a41a1638b3f","metric_details":{"license":"Proprietary","variance":15.327094333867809},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-sonnet-4-5-20250929-high-32k","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"3f8e97047884dc26e2f23f17f93cb2b685413c56149e714d71838fed50fe540a","result":{"confidence_high":1400.699868099494,"confidence_low":1385.3534155159882,"sample_count":13253,"score":1393.0266418077413,"score_display":"1393","source_stated_rank":91},"run_fingerprint":"142b49d764f6c1bd50454132ced50cfb99ba60446f393679ea142dd7441901e6","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_cf2150debee34b4d5f1bf4b9","provider_config":{"source_label":"high","token_budget":32000},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e80ebfc3b7c598a8d202aa5ca131ce5f82f05b8350e45eca4eb825dcb65c70f1","metric_details":{"license":"Proprietary","variance":13.612936013737645},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-sonnet-4-5-20250929","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"03cc4188cbb8d2e577948231dac609328d5b7f63e2bf8e02329227079c8a0941","result":{"confidence_high":1391.2238592659578,"confidence_low":1376.7610057265601,"sample_count":15722,"score":1383.992432496259,"score_display":"1384","source_stated_rank":94},"run_fingerprint":"facbaa5e22d0f8abee07852bb54bcc775a393a1954ca3fe8fcbb66833b96d294","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dedb7457e3ef28308b3be03c","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"fb5457af0040d08b326e699ec0e099d785d4cea222f144497e65a420c54e99f0","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"6e77311bf008b300cf58e79cbd50703bbef51747b0f59fb2361050cd3f79a81a","result":{"confidence_high":1392.86,"confidence_low":1378.13,"sample_count":15150,"score":1385.49,"score_display":"1385.49","source_stated_rank":93},"run_fingerprint":"43b0009f97607f700b13e4f01b90515d7cd4ce8efabd9590f80624bf72cdf2b5","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7b180c1e322ff1a94f071599","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f6ee8820fed67e0a8fbb0772b5b9a6307cc79d1b433025ae86d3045b1a9f8827","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"ec89876074787b4343e85caed455c416b2dc7bd739646f50439caa15ac87e753","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":46.7,"score_display":"46.7","source_stated_rank":81},"run_fingerprint":"ea751d83fbc62f640a00ac9d829da9c915d6a16dad2a1f3b75ed0a7d4c5d2b6f","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ccf30264fde8350df3c208b8","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":274,"at":"2026-01-04","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-04","status":"stale"},"measurement_id":"70622c7eecbb989a3e33c2c7d709fa019d765fcbee3af2d6403dba48d988779a","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":""},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"Claude 4.5 Sonnet","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929_unknown","source_row_date":"2026-01-04","source_scaffold":"MAYA","upstream_leaderboard_url":null},"run_count":null,"scaffold":"MAYA","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"f136f8f675707cd2a0e95926d6bfa64c0e7ae0a69bec999ee18a3bb9f704de8c","result":{"confidence_high":null,"confidence_low":null,"sample_count":89,"score":42.7,"score_display":"42.7","source_stated_rank":94},"run_fingerprint":"cb6b3fc1ff84026a6e72d6ee5a60c24038743a762062017185a6a0f4519bd526","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7595903dc018bedd481f339b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":335,"at":"2025-11-04","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-04","status":"stale"},"measurement_id":"e8ef0d54bf7132ad2f8d075c57e84b722bbd963e31e562f62f485ad84b7c2226","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"standard_error_points":2.9000000000000004,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":"2025-11-04","source_scaffold":"Claude Code","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Claude Code","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"c5b4bb2a6a4b4eef8a6eca3ea099a9fbe7abe15a021c90e06fc3c7ff391b0414","result":{"confidence_high":45.784,"confidence_low":34.416000000000004,"sample_count":89,"score":40.1,"score_display":"40.1","source_stated_rank":104},"run_fingerprint":"48510e9ce45f7555c8767288d3fe8051381b6d6995045067adab9a47f6e0f64f","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7595903dc018bedd481f339b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":336,"at":"2025-11-03","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-03","status":"stale"},"measurement_id":"5c269d78a71c487bc6d06bbc4ebbe3224f74260b0c98b14d0eb8c21dcb8bc620","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"standard_error_points":2.8000000000000003,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":"2025-11-03","source_scaffold":"Mini-SWE-Agent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Mini-SWE-Agent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"e777298a949221aec730165deccbcf07672688a66765e09da47ae903090c6118","result":{"confidence_high":47.988,"confidence_low":37.012,"sample_count":89,"score":42.5,"score_display":"42.5","source_stated_rank":100},"run_fingerprint":"ca4afbe3375de6d9076f41232d9f17eb0524da785572dbc358231471e6ccfd53","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7595903dc018bedd481f339b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":337,"at":"2025-11-02","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-02","status":"stale"},"measurement_id":"0ed3e22f601c295963dffba89fdfa58d8cc1ce58f02e3952eb07bb1605414149","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"standard_error_points":2.8000000000000003,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":"2025-11-02","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"ed7b483b82ea1106a5066e342775325bcb9129823d6404c19e3ffa60e0ce0655","result":{"confidence_high":48.088,"confidence_low":37.112,"sample_count":89,"score":42.6,"score_display":"42.6","source_stated_rank":97},"run_fingerprint":"66b92f5bf27d446d4370924ed4abe5f9830aa3a52a3a1ef5b90f74c2387b09cc","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7595903dc018bedd481f339b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":339,"at":"2025-10-31","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-10-31","status":"stale"},"measurement_id":"6b6399e648276c840f026e1b96341743e04a69a80546f85464a840922c82b264","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"standard_error_points":2.8000000000000003,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":5,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":"2025-10-31","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"651c18badcefd445a0d7c1d0a1e33fe2b26507cef79ff04d5d7350b03b15739b","result":{"confidence_high":48.288,"confidence_low":37.312,"sample_count":89,"score":42.8,"score_display":"42.8","source_stated_rank":93},"run_fingerprint":"ebe34ecae1b52fad01797b267a380e8e3998ccd56475982ded97a8d273c0ae7d","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7595903dc018bedd481f339b","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":223,"at":"2026-02-24","basis":"evaluation_at","evaluated_at":"2026-02-24","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"eadf29a7083e0b363db319e3a0c6833bfbd5e8ea84fea5e2bfe7f08ce0bc6461","metric_details":{"cost":4.04814,"pass_2":15.81,"pass_3":12.63,"pass_4":10.31},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"terminal","submission_slug":"claude-sonnet-4-5_sierra_2026-02-26","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"ff01b22ab20c5e42ea4cd21f5a09af629c356c4279c5f9921d42a8a43e64b968","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":25.26,"score_display":"25.3","source_stated_rank":null},"run_fingerprint":"9498edb2b9fe382c7cd897992af053916246213ec12e29bf287fe1c99c9656b4","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4a73c29927d902c07689647e","provider_config":{"source_id":"tau3-bench","source_label":"enabled"},"provider_mode_key":"enabled","raw_label":"enabled","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":292,"at":"2025-12-16T18:09:14Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-12-16T18:09:14Z","status":"stale"},"measurement_id":"b41a0884773e10ea1ba3d5d407b87f69d36f092f43c76560080a3781f0a59f6d","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":3.1,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=claude-sonnet-4-5 (thinking); createdAt=2025-12-16T18:09:14Z","source_row_created_at":"2025-12-16T18:09:14Z","task_pass_coverage_threshold_percent":75},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"claude-sonnet-4-5 (thinking)","source_reasoning_label":"thinking-unspecified","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"00b4051eba876b2e250e03eb77eb226884b4d82482f5f28944adca6ade4c4533","result":{"confidence_high":62.6,"confidence_low":56.4,"sample_count":1000,"score":59.5,"score_display":"59.5","source_stated_rank":19},"run_fingerprint":"c0c65c877ea9ddf6d390db40b31947fb7a6638e091ead1604fd5c27d36607710","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=claude-sonnet-4-5 (thinking); createdAt=2025-12-16T18:09:14Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_47645ffae78b747c6516878c","provider_config":{"source_id":"scale-mcp-atlas","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:03:18Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:03:18Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8013768229a3cc785b42af94b8f2abd61336905c5bf7a9e08793b3dd2fe792b5","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":4.31,"leaderboard_last_updated_at":"2026-04-12","license":"Proprietary","model_link":"https://www.anthropic.com/news/claude-sonnet-4-5","total_cost_usd":43.73},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"FC"},"run_count":null,"scaffold":"FC","tools_allowed":null},"protocol_fingerprint":"6a9c99bccbca11748bdf70a2bdd6eb864726c94bf4a029f84ab95898fe96d192","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.24,"score_display":"73.24","source_stated_rank":2},"run_fingerprint":"5138670fb30340f44ea041f2b385ac1d9423726f5b82fa67b2e7658de6b09e92","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a5edc006937c8df2d2cc0448","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"bfcl-v4","metric":"overall_accuracy","name":"Berkeley Function Calling Leaderboard V4","release_id":"release_7d2560723dba9d549cd10b25","split_or_window":"overall","unit":"percent","version":"v4-ede5081a24bc"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":38,"at":"2026-08-27T22:30:42Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:42Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"58674419497260c4c192ec7fb33a2da1034e729066380b6b9657aa1adb90581e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","freshness_caveat":"global-leaderboard-date-is-not-a-row-date","latency_mean_seconds":3.84,"leaderboard_last_updated_at":"2026-04-12","license":"Proprietary","model_link":"https://www.anthropic.com/news/claude-sonnet-4-5","total_cost_usd":47.82},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prompted-function-calling","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"bfcl_mode":"Prompt"},"run_count":null,"scaffold":"Prompt","tools_allowed":null},"protocol_fingerprint":"564fbea687964bbbeb4c2a4d92f977e2c00894a357d9ecf0f2b928b7737d37ca","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":24.9,"score_display":"24.90","source_stated_rank":89},"run_fingerprint":"e30946ee888fd6ab1222a90b031910d83172507329a224d2dace439e3e7820ba","source":{"content_hash":"a11091415b1a130fe4f1a328a29b593127fe7d662c6a347281f5d868daa9154d","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-08-28T22:03:18Z","homepage_url":"https://gorilla.cs.berkeley.edu/leaderboard.html","id":"bfcl-v4","last_fetched_at":"2026-10-05T12:33:31Z","license":"Apache-2.0","locator":null,"name":"BFCL V4","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://gorilla.cs.berkeley.edu/leaderboard.html"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a5edc006937c8df2d2cc0448","provider_config":{"source_id":"bfcl-v4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prompted-function-calling","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e9f32c9814475bcc6896e07f90cad2bdc644eaccb269c8dfa3f1a75be4c4f75b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.5","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929_unknown","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"990637a02b64bd8eac62a2e1fc535612686efac216914c730ecfc273dbc4f6ad","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3838.7400000000002,"score_display":"3838.74","source_stated_rank":39},"run_fingerprint":"91dae57948f50f132ab09cfdc196fc9ef9f1a1f190e5c0234aebdf59ac17643f","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f2188ef14be4e31531958a1b","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"27dbdb3ff07ca2d0df1572e6be18281ca037048a0af47bea7ab75277c8d229ca","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2025-09-29","reported_confidence_interval":"-26 / +26","source_locator":"Leaderboard models table · Claude 4.5 Sonnet (Reasoning) · gdpval"},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"claude-4-5-sonnet-thinking","upstream_model_label":"Claude 4.5 Sonnet (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"6ed21f5d43e23730ee5c0e11c639f97b5398a5acbd73db3a136162f25d69d818","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":892.87,"score_display":"893","source_stated_rank":null},"run_fingerprint":"30b7e4d84d0b6906ba4720dff8af08334f164394d6aa7b6ef6dfb7a3e75a3e26","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Claude 4.5 Sonnet (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_642e3a37222b1abf74918908","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval","metric":"win_rate","name":"GDPval","release_id":"release_730527fa346470703bd19e21","split_or_window":"expert-comparison-win-rate","unit":"percent","version":"GDPval official leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:05Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:05Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3265778f1463ea951f078209b6830bab873a1130b4b0d7aebe97b3857be94d9a","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-09-29","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-sonnet-4-5","name":"Claude Sonnet 4.5","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Sonnet 4.5","source_effort":null,"source_model_version":"claude-sonnet-4-5-20250929","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenAI GDPval expert-comparison protocol","tools_allowed":"as reported by the GDPval leaderboard"},"protocol_fingerprint":"739dfefac2e9651bb5c3941cdd027d7dc9b413f02dc8f80441112582eb9505cb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.5,"score_display":"42.5","source_stated_rank":4},"run_fingerprint":"dbfc9921f0d5301fd6c978f7098a7b625ded218342431aa0c84ea949669b7421","source":{"content_hash":"d4eac25e0863d96ee45cc545fc896e5667d7022da8b3bc3902533ed87fa8b836","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gdpval","id":"epoch-gdpval","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GDPval","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gdpval"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2ef94f4849290d1cb68eb0ba","provider_config":{"source_id":"epoch-gdpval","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"426bdf4e33758fb17f41817ed212f4a0c034309c4dc7408f511a3b86d5d12e1a","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · intelligenceIndex"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b","upstream_model_label":"Gemma 4 31B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"1e50ab05e9e1d849b2873dc95d221f83b2eaada88d6dcc93fab54176a5e5e082","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":14.66895,"score_display":"14.7","source_stated_rank":null},"run_fingerprint":"e823613461896fe3638d7497ed84a63e96eb4b28d449a117518920196b10f6b8","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_19bb152ad555330fda392da5","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":27,"at":"2026-09-08T00:34:38Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-08T00:34:38Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1c71649eb7760ad6b08bb6a75ba1a5141885abf14eeea20121838673483648b0","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · intelligenceIndex"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b-non-reasoning","upstream_model_label":"Gemma 4 31B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"f4cfefa1b960b705abdc4ecc87625a58b4e9459555668ad25b93072180714dc6","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.919882,"score_display":"13.9","source_stated_rank":null},"run_fingerprint":"697a2c69abb389ebd89ddcac2c8f444014b1a6cdc1ebb17620af5fbbf6ba5780","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e828e372c23c6a9ca863f8c4","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b6ef9ca54f8cf73da935df78fbd615481776caadf82e2e877098c79ade11286e","metric_details":{"license":"Apache 2.0","variance":14.42404845364731},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1460.2790078830114,"confidence_low":1445.3915113624566,"sample_count":6133,"score":1452.835259622734,"score_display":"1453","source_stated_rank":76},"run_fingerprint":"2e714e9e2b68273c4334ad3dfc8d5b623ffcb623cb969701a38e69b0b01c109f","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2a1871ad8f8b32a3d2496758","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5535b18f812b81a1c5de002963d7bd3986f4d337f5b97e946ce762dff64661e1","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · hle"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b","upstream_model_label":"Gemma 4 31B (Reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"85e9f41b11cece25ac1da39fd4d00e777ed4b4ad4a2b429805f7ed61b5549604","result":{"confidence_high":25.471299,"confidence_low":21.888397,"sample_count":2158,"score":23.632994,"score_display":"23.6","source_stated_rank":null},"run_fingerprint":"636cc43a75378cfb4a35165d1bbb9b89b91c7f4d7f3009444a82f1dad17c00c5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_19bb152ad555330fda392da5","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6abce6ece8bde2db5dca6c69fa4bd55fe2a4de737c5159abff88f8af6b8749fe","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · hle"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b-non-reasoning","upstream_model_label":"Gemma 4 31B (Non-reasoning)"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"edac3b7146b97262fda375cb3ffe871d55ffeed306a39bb3da0956ed05214726","result":{"confidence_high":13.246802,"confidence_low":10.521897,"sample_count":2158,"score":11.816497,"score_display":"11.8","source_stated_rank":null},"run_fingerprint":"b67616605f55a777c6b612cc4722b08df768bab79e8bd0553a8c3adb2fa41e18","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e828e372c23c6a9ca863f8c4","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"factuality","higher_is_better":true,"id":"simpleqa-verified","metric":"accuracy","name":"SimpleQA Verified","release_id":"release_ccacd6f679c9142b534364b4","split_or_window":"verified","unit":"percent","version":"task-v7"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":126,"at":"2026-06-01T11:15:20.646874Z","basis":"evaluation_at","evaluated_at":"2026-06-01T11:15:20.646874Z","first_observed_at":"2026-08-27T22:30:38Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"17555c5ff27592ec9f4887d749ab5ec70b2367ec8a7ab7e94f3a411ebedf183f","metric_details":{"confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"model_version_slug":"gemma-4-31b-it","task_name":"simpleqa_verified_task","task_slug":"/benchmarks/tasks/benchmarkler/simpleqa-verified-task","task_version":7},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"80ee7690245a67a4e25f4db6369059e8b8b8fe3bad89432648ac923f3b47c398","result":{"confidence_high":12.73420167465827,"confidence_low":8.633319692863097,"sample_count":null,"score":10.683760683760683,"score_display":"10.7","source_stated_rank":39},"run_fingerprint":"31684978048bc762ab5287852f66060aed5c6a0063322195254e2860391e63f8","source":{"content_hash":"68fd964e1585d5d012753bd331d4c92c025a619c5cf5fcad83b7aff3494e9fc4","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T18:20:57Z","homepage_url":"https://www.kaggle.com/benchmarks/deepmind/simpleqa-verified","id":"simpleqa-verified","last_fetched_at":"2026-10-05T12:33:30Z","license":"Dataset MIT; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"SimpleQA Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/deepmind/simpleqa-verified/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d495975af2f81492de3a7940","provider_config":{"source_id":"simpleqa-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6e585b2f17c83c5be4760c842d6924e39d8cd3273b6c8f0a49419fe6ba72c89c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · gpqa"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b","upstream_model_label":"Gemma 4 31B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"c5d580f1a9fb5c97205bd11a59b56eb46ce2398dc8b06f9c30f361a4ed836f9d","result":{"confidence_high":89.860993,"confidence_low":80.094854,"sample_count":198,"score":85.656566,"score_display":"85.7","source_stated_rank":null},"run_fingerprint":"37300b665a540c127583d53ef768298c730711424f087edfc4cc6f936855b8d9","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_19bb152ad555330fda392da5","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"8485a89c9971faf781c0f603465cfaa54223b77264b39eae516dcf82d3bc705c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · gpqa"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b-non-reasoning","upstream_model_label":"Gemma 4 31B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"d561602d4dcee3a538bc84805e6258bc1aed1499c1c69d683b51ab00e5e0c083","result":{"confidence_high":81.653815,"confidence_low":69.871737,"sample_count":198,"score":76.262626,"score_display":"76.3","source_stated_rank":null},"run_fingerprint":"64247a38e27725849d2c6ce3fafac6c0694db1245aa7d3262acde842117325f3","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e828e372c23c6a9ca863f8c4","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_db867ffb8417ed556374eda1","split_or_window":"diamond-five-shot","unit":"percent","version":"task-v4"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":38,"at":"2026-08-27T22:30:40Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:40Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7e545dd09fe1f3c9cb7379136696e5dbe5ddc59fa5dccf6369fc30548185367e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_is_symmetric":true,"result_case":"numericResult"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"model_version_slug":"gemma-4-31b-it","task_name":"gpqa_task_diamond_5s","task_slug":"/benchmarks/tasks/benchmarkler/gpqa-task-diamond-5s","task_version":4},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f789f033d15e4eabb794b930cb0b2ae111101f8c0239cc8f6ae9418f6cc9015b","result":{"confidence_high":91.14366218782509,"confidence_low":81.58361053944763,"sample_count":null,"score":86.36363636363636,"score_display":"86.4","source_stated_rank":11},"run_fingerprint":"85bc0b25a2fe1ce9a91824348e6ec5f1c40c53889a1e614de9e32bdc47ffc801","source":{"content_hash":"89757b1d7def0d8cb203c8d566bf1ab0257305154eae0bb0256e905e343abb53","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-09-19T02:26:09Z","homepage_url":"https://www.kaggle.com/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set","id":"gpqa-diamond","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark CC-BY-4.0; Kaggle leaderboard results redistributable with attribution (2026-09-02)","locator":null,"name":"GPQA Diamond (5-shot)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://www.kaggle.com/api/v1/benchmarks/open-benchmarks/gpqa-few-shot-diamond-set/leaderboard"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e75629513d1bb97e326a221d","provider_config":{"source_id":"gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"6f50c251122a4d260d51f34eb64e900aefecd733f52ceedf0f59a9a7587d01cf","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · mmmuPro"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b","upstream_model_label":"Gemma 4 31B (Reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"06553e5ee69e7a62adcb05e738ae03468fb5c6fb0c2c2c86438f3a86f8ce9d9a","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":73.410405,"score_display":"73.4","source_stated_rank":null},"run_fingerprint":"3fc81810c9a24370713bac7707c36d7f686ca2c92ba0573da1311704d149d38a","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_19bb152ad555330fda392da5","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"0feaa38d8c6fd7925a8eeedf0f0f4130c7e6b86a696e7b64e4c435fdb8b2bed1","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · mmmuPro"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b-non-reasoning","upstream_model_label":"Gemma 4 31B (Non-reasoning)"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"7dcf9ab534f6b522d98a96fb18c8daa051f5da4bdfbc1903b7df95197dfcaa90","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":70.346821,"score_display":"70.3","source_stated_rank":null},"run_fingerprint":"1504d0cf723dd74496cf0f57b1081aeae3e4b28c893c2eeae584a75611b626ac","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e828e372c23c6a9ca863f8c4","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"54ea44c9b862a451d8d294a76e89a5685d79abd28efdae39eace05eef1e51c58","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · critpt"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b","upstream_model_label":"Gemma 4 31B (Reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"81202de1051a0a8567930475a195bb4a33130a6bac3d77772d98d23b376efad9","result":{"confidence_high":7.658369,"confidence_low":0.252618,"sample_count":70,"score":1.428571,"score_display":"1.4","source_stated_rank":null},"run_fingerprint":"ecea84f5074e66ced1ff3ccef127ceface2e93bb8d3cf102dbd0cebefe33ef99","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_19bb152ad555330fda392da5","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b7c3dafe1a138aaf292f690c73f1c09035c613b7ca3d80761c209adc469e36b3","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · critpt"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b-non-reasoning","upstream_model_label":"Gemma 4 31B (Non-reasoning)"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"518a01aab9116ebe7a4a1efc004136c78756823daa3c05c63052a72e7729f734","result":{"confidence_high":5.202487,"confidence_low":0.0,"sample_count":70,"score":0.0,"score_display":"0.0","source_stated_rank":null},"run_fingerprint":"439c84b3ba4f845ce28111b52ad266ef68458c364365b7c647fcff2698046681","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e828e372c23c6a9ca863f8c4","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_1f8553cc12d8235844f1a578","split_or_window":"2026-04-15/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-29T23:53:02Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"929bd5982ea3d47d29cc9bb1c2c8461e12257f6b7edd487f5b820ed41052bd1b","metric_details":{"comparison_warning":"Official post-release SWE-rebench window; use only for same-window comparisons and never mix it into the canonical rolling-window rank.","comparison_warning_code":"swe-rebench-post-release-window","model_release_date":"2026-04-02","pass_at_5":38.46153846153847,"potential_contamination":false,"sem":2.0830867704194773},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-post-release-window","comparison_warning_code":"swe-rebench-post-release-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","supplements_range_key":"1772323200000:1778803200000","upstream_model_id":"Gemma 4 31B__tools","window_from":"2026-04-15","window_status":"supplemental-post-release","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"716d95b37f928778ed736abc0cc74aaf85e769a20276a1ac7fbf52aecae953dd","result":{"confidence_high":20.493106480278588,"confidence_low":12.327406340234237,"sample_count":null,"score":16.410256410256412,"score_display":"16.41","source_stated_rank":null},"run_fingerprint":"7fb574026c65a7c2dd145513da22ab30597fc39d6ee1375c86ef6fc6b323e56b","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_95be02b559104aaf55501db0","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-post-release-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_72d764965dd7baf34f983249","split_or_window":"2026-03-01/2026-05-15","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":143,"at":"2026-05-15T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-05-27","status":"stale"},"measurement_id":"9baa992e40e44e8157887dffa15a8cb647275ac6603e1283721ba4487353692d","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2026-04-02","pass_at_5":37.27272727272727,"potential_contamination":true,"sem":1.1281521496355322},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","potential_contamination"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","archived_publication":{"chart_url":"https://swe-rebench.com/insights/2026-05-27/leaderboard_with_logos_v7.png","evidence":"derived_from_chart_value_match","insight_id":"may_2026","published_at":"2026-05-27","range_key":"1772323200000:1778803200000"},"selection_rule":"dated-official-publication","upstream_model_id":"Gemma 4 31B__tools","window_from":"2026-03-01","window_status":"archived","window_to":"2026-05-15"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"662332765a9ebff9d2aaab450af1ab776ec950df89e06593ae06276be1656580","result":{"confidence_high":18.756632758740185,"confidence_low":14.3342763321689,"sample_count":null,"score":16.545454545454543,"score_display":"16.55","source_stated_rank":null},"run_fingerprint":"66cf164e2efad81b244d8f41b0a907427bbea2eed398aade8eb8f07e752cfbb3","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_95be02b559104aaf55501db0","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"C","evidence_rank":7,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"potential-contamination","verified_status":"potential-contamination"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"b2630205b3cdccbc4708f0faf8003dc8766aad88e9cbacd2b09fb2c46d38475e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · scicode"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b","upstream_model_label":"Gemma 4 31B (Reasoning)"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"020ae25233a02f3f7f0dddb3c1289de898bcdc8ea7e0ef6eeda9a03fc32de7a4","result":{"confidence_high":51.258985,"confidence_low":39.832072,"sample_count":288,"score":45.486111,"score_display":"45.5","source_stated_rank":null},"run_fingerprint":"c8a5014e6d927012ee54e745ced3dc9556edd5515855a83e8fc548b51454069f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_19bb152ad555330fda392da5","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"d6d7fa4e509d78ef32899662e82d3366cf7fb5293a46d3c9446440c6c5b7e802","metric_details":{"license":"Apache 2.0","variance":10.601976756615848},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemma-4-31b","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"a088506a3208c35b313de15c6cb5e721e8010e194ca8f395c1798718d32e2abe","result":{"confidence_high":1371.0597511153142,"confidence_low":1358.296200335301,"sample_count":13325,"score":1364.6779757253075,"score_display":"1365","source_stated_rank":100},"run_fingerprint":"a01e6fdbba0787a62018356f5079fe692e504d4f16076debc322c932dea12b69","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5156ce72059ef2337abe25ad","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"da0ae838f1cf6f8875791ea14137f8dc0665fb56821e153e4709f83d838e3cd2","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-02","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · terminalbenchV21"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"adaptive","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemma-4-31b","upstream_model_label":"Gemma 4 31B (Reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"ffb17a590195a0d2e41071c105dbdcc68d9b8046d7690713bce191b2707023a8","result":{"confidence_high":53.803572,"confidence_low":33.630223,"sample_count":89,"score":43.445693,"score_display":"43.4","source_stated_rank":null},"run_fingerprint":"c0c64167aca4193f1a49228d4e173b2c221b45935959e959646c5919818e1c8e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_19bb152ad555330fda392da5","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"40ad0cdb749f088d2b2d8bed2ba72ebfd2c684e3d6b4c37000d2c0ac935ff98b","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-02","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · terminalbenchV21"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"off","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"gemma-4-31b-non-reasoning","upstream_model_label":"Gemma 4 31B (Non-reasoning)"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"75b99f10dc041ec116d2a582234b757eb27e178c3c356e216c2b2af71f7fbfba","result":{"confidence_high":39.363701,"confidence_low":20.783474,"sample_count":89,"score":29.213483,"score_display":"29.2","source_stated_rank":null},"run_fingerprint":"e2a4f0d489b433ab71b676f52b711d06e0219b5365fbe554690969a95a3cf025","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e828e372c23c6a9ca863f8c4","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c43b0fc866937e3ba0d00345fb284d33dd7f5a9100a8b8c1de445a5b8845dbb8","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · tauBanking"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"adaptive","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b","upstream_model_label":"Gemma 4 31B (Reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"8882763b55ccea7fff2ae6ae063ae2ff1f9d8a2b7785d659f546b548e6a04104","result":{"confidence_high":23.252255,"confidence_low":9.116927,"sample_count":97,"score":14.845361,"score_display":"14.8","source_stated_rank":null},"run_fingerprint":"006b2348ee8f920acedda4b2f39226ecc1a81aabf06cf08b469bd9c2ba5c8d80","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_19bb152ad555330fda392da5","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"42ceb490ff80da9d7d616e5161b0782f71edd58d192422e1eb1ce66bc4d46d32","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-02","source_locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · tauBanking"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"off","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"gemma-4-31b-non-reasoning","upstream_model_label":"Gemma 4 31B (Non-reasoning)"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"de31ae5e085a9e9a691e6f696ab9f70297c85a760c0f3d2dcf95c984476de1f5","result":{"confidence_high":16.198091,"confidence_low":4.6679,"sample_count":97,"score":8.865979,"score_display":"8.9","source_stated_rank":null},"run_fingerprint":"9aa948083fa442aaa19af7b683bd069ff78cf52411a87bec63887c6c4198f2d5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e828e372c23c6a9ca863f8c4","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ea30cf58fb501172ec6c39f50f17c9954be158dcbf4143b2a15590682944a7ae","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-02","reported_confidence_interval":"-23 / +23","source_locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · gdpval"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"adaptive","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemma-4-31b","upstream_model_label":"Gemma 4 31B (Reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"955ee162634de246898d157ce060fb0a99a2b5d921161faef903baae9b439979","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":613.85,"score_display":"614","source_stated_rank":null},"run_fingerprint":"b8731063e178adba6ff3c6a08ca3934d2a0c4f602f7f7163c2b9c04d6fbcb3a1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_19bb152ad555330fda392da5","provider_config":{"source_label":"adaptive"},"provider_mode_key":"adaptive","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1e93dc7fd912b60fb08d24f644ce8306651267a4298409ae0c36b072fd4a597e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":false,"model_release_date":"2026-04-02","reported_confidence_interval":"-27 / +27","source_locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · gdpval"},"model":{"id":"google/gemma-4-31b","name":"Gemma 4 31B","provider":"Google","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"off","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemma-4-31b-non-reasoning","upstream_model_label":"Gemma 4 31B (Non-reasoning)"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"0356ec8468285faf22ab214917dd46e97811a9d2b763fd713f72f8b4c74d5204","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":535.29,"score_display":"535","source_stated_rank":null},"run_fingerprint":"324e3e3caeee641d29745500f6f25e16772fcfcea8d134866c83e48d091598e0","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemma 4 31B (Non-reasoning) · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"observed","effort_rank":0,"is_provider_default":false,"normalized_effort":"off","profile_id":"profile_e828e372c23c6a9ca863f8c4","provider_config":{"source_label":"off"},"provider_mode_key":"off","raw_label":"off","reasoning_enabled":false,"strategy":"none","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"5bb2083d06de74a60ce17e19609941dc384da530780c3acda94fa45493510145","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-03-03"},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 3.1 Flash-Lite; model release: 2026-03-03","source_accessibility":"API access","source_model_name":"Gemini 3.1 Flash-Lite","source_model_release_date":"2026-03-03","source_model_versions":["gemini-3.1-flash-lite","gemini-3.1-flash-lite_high","gemini-3.1-flash-lite_low","gemini-3.1-flash-lite_minimal"],"source_reasoning_efforts":["minimal","low","high"]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0be7fc5bd284bb44ef1d7b0f96b123b70f42eb3998d1a9af09ff29962fcf95ad","result":{"confidence_high":146.15,"confidence_low":142.37,"sample_count":null,"score":144.45,"score_display":"144.45","source_stated_rank":85},"run_fingerprint":"54ac24cc44b0709eb2079cb1b7c9a4458dcdc64417ebd5110dab73dfa56b83c0","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_b8b4ba91c6c80d65c7545f45","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ba050edb7adb607527edac50bcf7ea19423914cf05c4069ba1d5185db6af9922","metric_details":{"license":"Proprietary","variance":3.4349192294854047},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1436.3821697713895,"confidence_low":1429.1171575146955,"sample_count":64047,"score":1432.7496636430426,"score_display":"1433","source_stated_rank":109},"run_fingerprint":"3e89ff95cf19e5559bb21e1e1a2e371f556b00759b2ddfa3bc23d13ebf885c42","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_85184dd795400dea9b90c76f","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle","metric":"accuracy","name":"Humanity's Last Exam · Full multimodal","release_id":"release_8817af9d9a0235bce0826e22","split_or_window":"full-multimodal-2500-no-tools","unit":"percent","version":"HLE finalized 2,500 · Scale protocol 2025-04-03"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":195,"at":"2026-03-23T21:13:29Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-09T22:25:06Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-03-23T21:13:29Z","status":"stale"},"measurement_id":"dadd8f51ff23fa110a2bf1a48946eaaf9fc117b41569128b7aa7728198ebfdad","metric_details":{"calibration_error":83.0,"confidence_interval_half_width":1.1,"max_score":58.4422,"rank_method":"rank by confidence-interval upper bound","source_contamination_note":"Potential contamination warning: This model was evaluated after the public release of HLE, allowing model builder access to the prompts and solutions.","source_locator":"embedded leaderboard row: model=gemini-3.1-flash-lite-preview; createdAt=2026-03-23T21:13:29Z","source_row_created_at":"2026-03-23T21:13:29Z"},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"o3-mini-2025-01-31","run_config":{"judge_temperature":0.0,"multimodal":true,"protocol":"finalized full multimodal HLE over all 2,500 public questions","protocol_owner":"Scale AI Labs and Center for AI Safety","public_task_count":2500,"source_model_label":"gemini-3.1-flash-lite-preview","source_transport_sha256":"30db046f517eab19f9849783fcabd62fe41c5a290f1bb00995136b0a880f0053","temperature":0.0,"temperature_policy":"0 when configurable unless row note states otherwise","tools":"none"},"run_count":null,"scaffold":"Scale HLE evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"7a238bdae66d9bdf25075665dfb9e7fb7b9cc3b2208d8e68fd58fe2df87cde29","result":{"confidence_high":9.74,"confidence_low":7.540000000000001,"sample_count":2500,"score":8.64,"score_display":"8.64","source_stated_rank":30},"run_fingerprint":"19789d4818d3571e6fdef43f1d18e0109249d9c6d7f156b15c565ac1b7cd16d5","source":{"content_hash":"1c198fab689fb23a25daa60c4551cc62ba1ad938fd6dd9515209cf22f2cc83e8","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-09T22:25:06Z","homepage_url":"https://labs.scale.com/leaderboard/humanitys_last_exam","id":"scale-hle","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; HLE dataset terms apply","locator":"embedded leaderboard row: model=gemini-3.1-flash-lite-preview; createdAt=2026-03-23T21:13:29Z","name":"Scale AI Labs · Humanity's Last Exam Full","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/humanitys_last_exam"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fae6ffed555320b9ca4122c0","provider_config":{"source_id":"scale-hle","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"95b6548e270c79e67a0d6abcce6f34844a643f8ce8a9c084d19277efbd2a9717","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.002298,"date_is_proxy":true,"latency_seconds":16.464,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":1.967},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":79.293,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":65536,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-flash-lite-preview","zero_shot_accuracy":82.828},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"f54470c019ec3433be1a986fb642b8873cb217b9f975a3652fd804255151b472","result":{"confidence_high":84.91532000000001,"confidence_low":77.20468,"sample_count":396,"score":81.06,"score_display":"81.06","source_stated_rank":66},"run_fingerprint":"95c662a7c941f93a658a4cf910029144c315291603c01a91374721bfc3375149","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:26:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:26:07.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"dd73e202bbf3d62cadacedfb116e9e9f0e94dc9830779e960a3e9f61123aa8c4","metric_details":{"best_score_across_scorers":"0.8181818181818182","model_release_date":"2026-03-03","stderr":2.7479603010538827},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"nE4hmJo3QV6X5jeGWAeAze","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnE4hmJo3QV6X5jeGWAeAze.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/nE4hmJo3QV6X5jeGWAeAze.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"0a002b1cac8481c8bc2bd6ec3a4855057b226715e1e4a80e5d6d377b202dc0b1","result":{"confidence_high":87.20418400824744,"confidence_low":76.43217962811622,"sample_count":null,"score":81.81818181818183,"score_display":"81.8","source_stated_rank":118},"run_fingerprint":"43ebcd9d79f3a27081567343af6b78bf0d9f482c6dc4a11c84a14d5cf86b5250","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:25:58.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:25:58.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b4542342c3d72d195b875007568acea3ff2fe3349eaebf7834e770c226004481","metric_details":{"best_score_across_scorers":"0.7424242424242424","model_release_date":"2026-03-03","stderr":3.115626951964683},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"99ys2jgQiPQ5UwrqnazfqA","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F99ys2jgQiPQ5UwrqnazfqA.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/99ys2jgQiPQ5UwrqnazfqA.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"25033daa8d8a6a87a6bdcb4e46d67ea29e25b56bb91942d3675476a3d8ef1617","result":{"confidence_high":80.34905306827503,"confidence_low":68.13579541657347,"sample_count":null,"score":74.24242424242425,"score_display":"74.2","source_stated_rank":162},"run_fingerprint":"f981ef35a6e545447193853c938b714d3a90b7dcc57eba1e659474cbb16bbcc4","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_614eea4dff70de5dca23be32","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":59,"at":"2026-08-06T22:26:14.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:26:14.000Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"38e4776fce4b81141106a59ab6bc606ee8912264d04c5b1a660cb4fb41560774","metric_details":{"best_score_across_scorers":"0.7373737373737373","model_release_date":"2026-03-03","stderr":3.1353050095330834},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CmHy8cVECGkvQTwakfjRo2","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCmHy8cVECGkvQTwakfjRo2.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/CmHy8cVECGkvQTwakfjRo2.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"4a4090b4d63112c6f1abc8ec768688f2954282944deed45c8c3de87103ff7a1e","result":{"confidence_high":79.88257155605858,"confidence_low":67.59217591868888,"sample_count":null,"score":73.73737373737373,"score_display":"73.7","source_stated_rank":165},"run_fingerprint":"45f8a06f4a33725241bea7498a4a19f1b924b508df1fe838259b5c678b8db80f","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_ce5fdd36814b0f4571fdf033","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"80c1b68e074ab53d635a44981399ce4b9ec0ff276659f9479f49930113406471","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.001052,"date_is_proxy":true,"latency_seconds":14.511,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.34},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":null,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":null,"top_p":null,"upstream_model_id":"google/gemini-3.1-flash-lite-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"d35cd447d41d28e62beb9b9915c45f7ac015e4a9dfd12d84bd2e41991681a492","result":{"confidence_high":86.90639999999999,"confidence_low":85.5736,"sample_count":null,"score":86.24,"score_display":"86.2","source_stated_rank":45},"run_fingerprint":"20cf4762c1e8889ee13a342ca19de8ebd2d2159e45a6ef0553387d122d0dc984","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_af3cb42bfe3d70deaa28ebb5","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:29:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:29:57.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"940f3b542c6f2bd1b64914b255bef4c0189aba34d65a8fa0adbc0ead59486a06","metric_details":{"best_score_across_scorers":"0.8","model_release_date":"2026-03-03","stderr":6.030226891555273},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"NMdaAXyqqPRmWfuxZUfHFF","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNMdaAXyqqPRmWfuxZUfHFF.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/NMdaAXyqqPRmWfuxZUfHFF.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c559d91aa907d379896419ae84e6b65fff9996bd6b2f0392f3252b4613fb2f0b","result":{"confidence_high":91.81924470744833,"confidence_low":68.18075529255167,"sample_count":45,"score":80.0,"score_display":"80.0","source_stated_rank":110},"run_fingerprint":"87cf0d5d776c6f51112892ce3aef4fc5f0fc06bcc834627944aa1d04bd5d40f7","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:30:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:30:07.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bb87e45c048a058d383c521e5f4147ec4fb5ccd315e7e459335cfa1319f9fd25","metric_details":{"best_score_across_scorers":"0.4444444444444444","model_release_date":"2026-03-03","stderr":7.4911095829249135},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"CFBCC2QrkCAFBAvFj5pqPF","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCFBCC2QrkCAFBAvFj5pqPF.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/CFBCC2QrkCAFBAvFj5pqPF.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"082b8d01480c5392c15e5b7abc36d872e182ea832c412892c72c31892927faaf","result":{"confidence_high":59.127019226977275,"confidence_low":29.76186966191161,"sample_count":45,"score":44.44444444444444,"score_display":"44.4","source_stated_rank":200},"run_fingerprint":"2d5d185ee91484657021315826e8fc3cf733a45a750b59c68f341ed7844dabb4","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_614eea4dff70de5dca23be32","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:30:17.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:30:17.000Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"1ea8d77a304538bec76902fe00315d6e93ae989efd508fbbb2fcb609ea5b948e","metric_details":{"best_score_across_scorers":"0.37777777777777777","model_release_date":"2026-03-03","stderr":7.309112127323451},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"EoSFpgFhJB7iDPLcMNuLH4","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEoSFpgFhJB7iDPLcMNuLH4.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/EoSFpgFhJB7iDPLcMNuLH4.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"34fd6681ad80a455c0655c385838d7b26babf41e81fd44908188f1e4d585a9bf","result":{"confidence_high":52.10363754733174,"confidence_low":23.451918008223814,"sample_count":45,"score":37.77777777777778,"score_display":"37.8","source_stated_rank":209},"run_fingerprint":"565f2d955ff5e09bd1e2835669e9e08ada327640a160cc9f0206b6d65da39aac","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_ce5fdd36814b0f4571fdf033","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:34:53.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:34:53.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"0e59e83ae238580659dce2935f8c61bbdef94bf99c4eae87c89dbf398d18cb2d","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.2771929824561403","leaderboard_private_problem_count":285,"model_release_date":"2026-03-03","public_example_count":10,"stderr":2.6560920735701257},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"MVZf59jt9uEoWG9w5W9apB","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMVZf59jt9uEoWG9w5W9apB.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/MVZf59jt9uEoWG9w5W9apB.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.1 Flash-Lite","thinking":null,"upstream_model_id":"gemini-3.1-flash-lite_high","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"76a41eb19d2cdb9323ed468d84916f7a5baf86af23e43721843c12f3bcfd8bbe","result":{"confidence_high":32.92523870981148,"confidence_low":22.513357781416587,"sample_count":285,"score":27.719298245614034,"score_display":"27.7","source_stated_rank":73},"run_fingerprint":"e7b4f6d6c8604acce20a5e0b3f086b1c956c61eecbd39a1af41cb2fb9d8ae1b4","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":35,"at":"2026-08-30T22:35:03.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-30T22:35:03.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"b5876b6f6bee73828d586f315a302eacc6827d41a3d68f537e7d98e34290e915","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.22456140350877193","leaderboard_private_problem_count":285,"model_release_date":"2026-03-03","public_example_count":10,"stderr":2.4761793289496805},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"VcuyQvF8LWuYhQCx2FGDHh","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVcuyQvF8LWuYhQCx2FGDHh.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/VcuyQvF8LWuYhQCx2FGDHh.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.1 Flash-Lite","thinking":null,"upstream_model_id":"gemini-3.1-flash-lite_low","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"5bfee843175ca4070e1d784e10ea1eb12cb1388a492278dcee84afbc63cff495","result":{"confidence_high":27.309451835618567,"confidence_low":17.602828866135816,"sample_count":285,"score":22.45614035087719,"score_display":"22.5","source_stated_rank":80},"run_fingerprint":"b4aacd0e426e26fbf51414360efe41782c617774e172867dcbb92dbeda4012ac","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_614eea4dff70de5dca23be32","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-28T12:55:30.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-28T12:55:30.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8a277a40f6a46772181bc4fc438cbcffac4e10614df1a917f5dbb3c1126004f5","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.21403508771929824","leaderboard_private_problem_count":285,"model_release_date":"2026-03-03","public_example_count":10,"stderr":2.433800055326239},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"4gZy2dAAT2EPZDGz5puD3H","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4gZy2dAAT2EPZDGz5puD3H.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4gZy2dAAT2EPZDGz5puD3H.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"Gemini 3.1 Flash-Lite","thinking":null,"upstream_model_id":"gemini-3.1-flash-lite_minimal","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"db46216bb97b0e3cb55df34bb9752f0d65123248607dadc2ee7a48a5498ecf10","result":{"confidence_high":26.173756880369254,"confidence_low":16.633260663490397,"sample_count":285,"score":21.403508771929825,"score_display":"21.4","source_stated_rank":82},"run_fingerprint":"f6b716781ada12fe0c1956fdb34e33f3da090b35ddc669940c6c85dd4a89447f","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_ce5fdd36814b0f4571fdf033","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"c9ff4a4ba5ec31e4dff5604f6cd53234a31fbd7f39b3c152ac278bb6813d023e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.003187,"date_is_proxy":true,"latency_seconds":12.448,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.914},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":65536,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-flash-lite-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"3767ab56b37371d2813bfd58065b14e50ca8c7e205b4d1f26a02aeeffabbb938","result":{"confidence_high":84.27744,"confidence_low":80.69456000000001,"sample_count":null,"score":82.486,"score_display":"82.5","source_stated_rank":37},"run_fingerprint":"e5cca7e52b3abad85767628616fa689ac7665493a704ae9d2c2e7df657aff9d2","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"780d0ba21e1890cc17186ad9cd17e0d334b11342a6a5cc9cef45d483f6b4c329","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-03","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Flash-Lite","source_effort":null,"source_model_version":"gemini-3.1-flash-lite","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"d97854996351af878c366096ceac174b4dfc8204d84267a309a3c3f2d4c28daf","result":{"confidence_high":7.127519,"confidence_low":0.173844,"sample_count":71,"score":1.1428571428571401,"score_display":"1.1","source_stated_rank":140},"run_fingerprint":"72e2e65edd122110a0c98e4fd89f0d7fce827342e8d4083fb08770c529bb28ec","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e0159beebd90ac46fd46882a","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:26:13.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:26:13.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"37b1d8cb455d9aadb3cfd79cebe74fbf418d97fe028c909692b679854c5c7b84","metric_details":{"best_score_across_scorers":"0.2","model_release_date":"2026-03-03","stderr":4.020151261036848},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"YFmtuHAcUZeGwPBe9Jbcxw","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYFmtuHAcUZeGwPBe9Jbcxw.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/YFmtuHAcUZeGwPBe9Jbcxw.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"eca7f0450743c7c243efd53cdaf6d333f0f5804ac91edf56206755d3dc4d03fe","result":{"confidence_high":27.879496471632223,"confidence_low":12.120503528367777,"sample_count":100,"score":20.0,"score_display":"20.0","source_stated_rank":87},"run_fingerprint":"20137c1db955ba3214b2c1d2e97ddde64eb6e6a70074241ef595cafdadbe0be5","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:29:57.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:29:57.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3c5095b3c37dbe7de7d893aa9ddb1bb9cc4bdd55fe3e8a2c70894a3e69d77b54","metric_details":{"best_score_across_scorers":"0.25","model_release_date":"2026-03-03","stderr":4.351941398892446},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"nSLGeLNawfzhCbR33agwer","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnSLGeLNawfzhCbR33agwer.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/nSLGeLNawfzhCbR33agwer.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"9cabaa62e3eb0f5924c4e59facbfc4b6bb8412d11eaeeb435a4d807bf693ddc9","result":{"confidence_high":33.52980514182919,"confidence_low":16.470194858170807,"sample_count":100,"score":25.0,"score_display":"25.0","source_stated_rank":66},"run_fingerprint":"734c6d70953bfb9462d2eff9f1695934ae4aa2944e4234cdba5f97681acc371a","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":2,"is_provider_default":false,"normalized_effort":"low","profile_id":"profile_614eea4dff70de5dca23be32","provider_config":{"source_label":"low"},"provider_mode_key":"low","raw_label":"low","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":59,"at":"2026-08-06T22:30:01.000Z","basis":"evaluation_started_at","evaluated_at":"2026-08-06T22:30:01.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"534e341ee6b5905217eebb215d6a2dd05a39a894d90acd284c420cdfe98a9fed","metric_details":{"best_score_across_scorers":"0.24","model_release_date":"2026-03-03","stderr":4.2923469599092785},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"LurNQWcpKjLWQ7AXSD6Ab3","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLurNQWcpKjLWQ7AXSD6Ab3.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/LurNQWcpKjLWQ7AXSD6Ab3.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"10d107fc7440a702737e40c501dd7a3b40e715feb49193c338cca664bc977801","result":{"confidence_high":32.41300004142219,"confidence_low":15.586999958577815,"sample_count":100,"score":24.0,"score_display":"24.0","source_stated_rank":69},"run_fingerprint":"7dd9426707477d591bdb3f219e282401bea5e245647d2242aa05c264e50254f1","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"observed","effort_rank":1,"is_provider_default":false,"normalized_effort":"minimal","profile_id":"profile_ce5fdd36814b0f4571fdf033","provider_config":{"source_label":"minimal"},"provider_mode_key":"minimal","raw_label":"minimal","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5cd89da3d534d427edc8256e46fc95496d87ce8ee16c007b092922f53477a6fc","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.105389,"date_is_proxy":true,"latency_seconds":132.936,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.164},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-flash-lite-preview"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"282d7aabdc09b379093ce35e502e075d1e701f61095d40337f3715bccd9489a6","result":{"confidence_high":67.04144,"confidence_low":58.55856,"sample_count":500,"score":62.8,"score_display":"62.8","source_stated_rank":75},"run_fingerprint":"ea685f24e63bac4667e969adcd0f9b6960dd4d551007f21d368b142f53bbcacb","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"181fd798810feb80fac7787cd3c1da5158a450971a62ab40dd87907582872e20","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.00435,"date_is_proxy":true,"latency_seconds":30.749,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.083},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":null,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":null,"top_p":null,"upstream_model_id":"google/gemini-3.1-flash-lite-preview"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"408b3ce7fe671ecebeed64498dc80c07920f7ca2f2d09e94042fca78d57c587d","result":{"confidence_high":82.23868,"confidence_low":77.99332,"sample_count":null,"score":80.116,"score_display":"80.1","source_stated_rank":71},"run_fingerprint":"f84ae0b0e1f87a0c5cd0f36afc2a26d2e373156d9b68ec16eb1c8a356eaff5fb","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1fef4121bfa2191159e58ad8","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"ff1de937b1171ed8fe43ef57d3fdf16b0af2a4b0332002927be305598c504894","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-03","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Flash-Lite","source_effort":null,"source_model_version":"gemini-3.1-flash-lite","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"8d3c86b7598bc679872226ba49e2faa9c19214088908e0d230c1ac3320f40589","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":41.8981481481481,"score_display":"41.9","source_stated_rank":140},"run_fingerprint":"e319f99a5ed5248876b151e60fe54e4064178f34d6b78e9e90cbf7b0350c5861","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8a65ab7f90e126b7366a0131","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"ale-bench","metric":"performance_rating","name":"ALE-Bench","release_id":"release_a4c08439e33767077f5fed9a","split_or_window":"atcoder-heuristic-contest-set","unit":"points","version":"ALE-Bench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:01Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:01Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3959ec21bb4665c43c63b13145a81f042f7b87ebfae623dce8e82bcac5751726","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost":"0.025178","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-03","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"gemini-3.1-flash-lite-preview-high","source_effort":null,"source_model_version":"gemini-3.1-flash-lite","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"ALE-Bench official evaluation","tools_allowed":"code execution against contest scorers"},"protocol_fingerprint":"c7d23156106494bfa181950a8966e70af57053e2590f4d96cb6d0b63956fbabf","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":797.73,"score_display":"797.73","source_stated_rank":67},"run_fingerprint":"c177f1f9080b50d11fc082ad09bac8430d08451a2b83169ee77cf276ad8374d3","source":{"content_hash":"0e2fbb929d1a8ddfb6183e6a0407eb859ee15023ab4dfbcf193f3105c81b978c","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/ale-bench","id":"epoch-ale-bench","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ALE-Bench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/ale-bench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1e26a03d71f7608dc6a2e90b2d51044e0075b0802cdc6d0db0c9b9afbe53cfc6","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.51,"date_is_proxy":true,"latency_seconds":572.711,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.0},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-flash-lite-preview"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"f99f47d0988af377ab404a188f0159190beead3a164d28a83f58fd9047ee20fd","result":{"confidence_high":0.0,"confidence_low":0.0,"sample_count":null,"score":0.0,"score_display":"0.0","source_stated_rank":105},"run_fingerprint":"bb4d2004ef50ea3c2455cbdb7250a80986877af893ec9edead8f7f32b0b453c7","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ef88c5949dded1384d6fef91553366b9cc7291a2d00c61bc22db021d4a13f999","metric_details":{"license":"Proprietary","variance":12.069322603084938},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-3.1-flash-lite-preview","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6a43f9e8560d5f0db5f1643dda36beb281134019798fdfcf13fb7eebb490fe85","result":{"confidence_high":1262.4719932447824,"confidence_low":1248.85379868406,"sample_count":14417,"score":1255.6628959644213,"score_display":"1256","source_stated_rank":123},"run_fingerprint":"c59ce4a9ae73f3a42698db2b504d6093120fa5bd289de939e92b82f5e96bb7d6","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ee441737c431e7aa925a4c16","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"9f679b92764f944a8337e233e2644c152db6f470988e4a1e2a1bf4db1e796b63","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-03","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.1-flash-lite","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"98e98578cbad638d1888d2d59db75f1ccdc2c22815d972d94ff44363001abb4f","result":{"confidence_high":1261.24,"confidence_low":1247.29,"sample_count":13634,"score":1254.27,"score_display":"1254.27","source_stated_rank":120},"run_fingerprint":"875eebaa1cef466f6878b8097d4ddf1e9260b3e091fb3a0d5ce9bd9dd958e836","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8547bd48b3f3e6b6a7333919","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"d9d082287ad3646a369d32fecbe8ee874ab39dbd542b41d3a896af4fb4e825f7","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2026-03-03","notes":null,"upstream_source_url":"WeirdML Leaderboard"},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-3.1-flash-lite","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"b776e2ee0a3bbfc2473e30ae3be99d4a2a44a241ede849ea7d350b24d3d312c3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":52.190000000000005,"score_display":"52.2","source_stated_rank":69},"run_fingerprint":"4c0631f656e79291de212fdfd993233c3c40010c205ed29994b1890b5b8039d5","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f4e3bdd09734bbd928be4cde","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"41cf88e33896061163ed32207aa3c52d87a521505427bd91764e98ef25f0f8d9","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.084894,"date_is_proxy":true,"latency_seconds":278.342,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.991},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":65536,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-3.1-flash-lite-preview"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"12c173ca1fd4917058ee3e77fe034e8187e42ec89e3890611af3ab52fdae07d0","result":{"confidence_high":36.02436,"confidence_low":32.13964,"sample_count":null,"score":34.082,"score_display":"34.1","source_stated_rank":72},"run_fingerprint":"99d3fea1b050d718e64d40b200111c3efc6c7fe615d82d30927a98a24fb1496e","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"mcp-atlas","metric":"pass_rate","name":"MCP-Atlas · Scale","release_id":"release_6481c5033c8c5bc6dab70199","split_or_window":"all-1000-public-500-private-500","unit":"percent","version":"MCP-Atlas April 2026 rescored protocol"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":269,"at":"2026-01-08T19:31:06Z","basis":"submission_at","evaluated_at":null,"first_observed_at":"2026-09-21T20:27:30Z","last_confirmed_at":"2026-10-05T12:33:23Z","observed_at":"2026-10-05T12:33:23Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-08T19:31:06Z","status":"stale"},"measurement_id":"10a0672d38366a6d2b377ef3426492d0575110faec079abd72c94ffdfc208bad","metric_details":{"claim_scores":[0,0.5,1],"confidence_interval_half_width":3.0,"max_score":92.7515,"rank_method":"rank by confidence-interval upper bound","source_locator":"embedded leaderboard row: model=gemini-3.1-flash-lite (high); createdAt=2026-01-08T19:31:06Z","source_row_created_at":"2026-01-08T19:31:06Z","task_pass_coverage_threshold_percent":75},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":"Gemini-2.5-Pro","run_config":{"behavior_shaping_system_prompt":false,"judge_temperature":0.0,"max_tool_calls_per_task":100,"mcp_server_count":36,"model_configuration":"provider default","private_task_count":500,"protocol_owner":"Scale AI Labs","protocol_revision":"April 2026 rescoring","public_task_count":500,"source_model_label":"gemini-3.1-flash-lite (high)","source_reasoning_label":"high","source_transport_sha256":"88da72da9ef61cedce980c74c0823236ea98a55e4ee774d2337e2f6c9fde5b99","task_pass_coverage_threshold":0.75,"tool_count":220},"run_count":null,"scaffold":"Scale MCP-Atlas containerized harness","tools_allowed":"36 MCP servers / 220 tools / max 100 tool calls per task"},"protocol_fingerprint":"775dada544bc872b5094d6d9589bd85825ce45f4307102fb3cb997e1da181930","result":{"confidence_high":60.1,"confidence_low":54.1,"sample_count":1000,"score":57.1,"score_display":"57.1","source_stated_rank":19},"run_fingerprint":"71dee9bad9f4c850a5693045cd73aa5c3d685d78ac532fa875955821a8ee3449","source":{"content_hash":"a4714da76a70d479c7529a0449557cc61673046bf765daf0abd94a09e19cc643","fetched_at":"2026-10-05T12:33:23Z","first_fetched_at":"2026-09-21T20:27:30Z","homepage_url":"https://labs.scale.com/leaderboard/mcp_atlas","id":"scale-mcp-atlas","last_fetched_at":"2026-10-05T12:33:23Z","license":"Aggregate leaderboard facts with Scale AI Labs attribution; benchmark artifacts retain their upstream terms","locator":"embedded leaderboard row: model=gemini-3.1-flash-lite (high); createdAt=2026-01-08T19:31:06Z","name":"Scale AI Labs · MCP-Atlas","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://labs.scale.com/leaderboard/mcp_atlas"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_5d942d01fc59a4928ff4488d","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"submission_at","date_rank":3,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_093b45b8b752b3f08534f6ee","split_or_window":"official-87-tasks-with-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"0ad8dd70f4378b03889d84674613296930c8dd5d9f7ae776ad986ad5b63845ce","metric_details":{"expected_trials_per_mode":261,"normalized_gain":4.9,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":4.1},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"with-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools plus benchmark skills"},"protocol_fingerprint":"c2ed2025e534826e5e627446f4493a128269710491b35eb2122abbaf19d56304","result":{"confidence_high":26.900000000000002,"confidence_low":13.3,"sample_count":87,"score":20.1,"score_display":"20.1","source_stated_rank":21},"run_fingerprint":"107cfd6d5439b2207c1b21b463a7fc2fb86e06663624c53284bcff59de65934d","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_08bfcf047d9c1196e9090da8","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"skillsbench","metric":"pass_rate","name":"SkillsBench v1.1","release_id":"release_7f4a4905a388802e2e95ec03","split_or_window":"official-87-tasks-without-skills","unit":"percent","version":"v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":111,"at":"2026-06-16","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T19:57:35Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-06-16","status":"stale"},"measurement_id":"e9724084a2ecb9abcc933bcb52c8656095d4336f6ce02b39768bff15ac633430","metric_details":{"expected_trials_per_mode":261,"normalized_gain":4.9,"recomputed_date":"2026-06-11","selected_trials":261,"skill_lift":4.1},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-no-skills","comparison_warning_code":null,"judge":null,"run_config":{"agent":"openhands","audit_status":"reviewed_release_aligned","coverage_status":"full","harness":"benchflow","selected_trials":261,"skill_mode":"without-skills","source_commit":"27738384b1df694ea2ae466e416f476e94d8fab9"},"run_count":3,"scaffold":"openhands","tools_allowed":"agent tools; benchmark skills withheld"},"protocol_fingerprint":"e2fff9540af08b3f3233b880ba9f91ca1e57f7b3190c84dd9a4000ce24c7aca1","result":{"confidence_high":22.3,"confidence_low":9.7,"sample_count":87,"score":16.0,"score_display":"16.0","source_stated_rank":20},"run_fingerprint":"6ebcaa7709cd4061840d8ad73da13baabfaac3feae4faebebf2ebb818c5852b8","source":{"content_hash":"89f65fa6af85c89fbf2bb9af573367ff7a0dcd03b2b96b930bd25f1af5d78b0d","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-08-28T19:57:35Z","homepage_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard","id":"skillsbench","last_fetched_at":"2026-10-05T12:33:28Z","license":"Apache-2.0","locator":null,"name":"SkillsBench v1.1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/benchflow/skillsbench-leaderboard/raw/main/leaderboard/skillsbench/v1.1/official.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_08bfcf047d9c1196e9090da8","provider_config":{"source_id":"skillsbench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-no-skills","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"97d4aff950088d944d879f40971d1753b4880980ecf1012f19491ee2a910d80b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.25","mean_standard_error":"2.5","model_release_date":"2026-03-03","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":2.2,"upstream_source_url":null},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 3.1 Flash Lite","source_effort":null,"source_model_version":"gemini-3.1-flash-lite","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"7890fc5221d5f4a05faed66163b6cbdf4a7778f98d3da1a3b3787c3a66d574cc","result":{"confidence_high":17.312,"confidence_low":8.687999999999999,"sample_count":480,"score":13.0,"score_display":"13.0","source_stated_rank":49},"run_fingerprint":"11202a1e1825b728deb8100e068d85cbd665abbaf10511f50459c98b5fcdecca","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_739c7cb95867d31de84c5a08","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"fcad2939022e582220fdc499eedc751949585fb976b54e0f2766e84ba870f44b","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2026-03-03","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Gemini 3.1 Flash-Lite · gdpval"},"model":{"id":"google/gemini-3-1-flash-lite","name":"Gemini 3.1 Flash Lite","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-3-1-flash-lite-preview","upstream_model_label":"Gemini 3.1 Flash-Lite"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"2bca66dd288faa587ae59b3aaf5af7023e1afe04181a7382034a4ffe407d0d06","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":416.1,"score_display":"416","source_stated_rank":null},"run_fingerprint":"e3a153a262865082bcb9e4e062ae56d1083e03f1b006fb37111c35c0881fbae7","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 3.1 Flash-Lite · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ae6c97fa1dba1a92fb1bb243","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"f7dc49327beadaf06724ae55b021766adfb0b04aaaf50b50d245999612477c7e","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-08-05"},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Claude Opus 4.1; model release: 2025-08-05","source_accessibility":"API access","source_model_name":"Claude Opus 4.1","source_model_release_date":"2025-08-05","source_model_versions":["claude-opus-4-1-20250805","claude-opus-4-1-20250805_16K","claude-opus-4-1-20250805_24K","claude-opus-4-1-20250805_27K","claude-opus-4-1-20250805_32K","claude-opus-4-1-20250805_unknown"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8dd43ea403672b6d565b5d6f5ed5460f2d0eefeb4cd8e539e818a373af4250e9","result":{"confidence_high":146.04,"confidence_low":141.64,"sample_count":null,"score":144.11,"score_display":"144.11","source_stated_rank":88},"run_fingerprint":"5026630afa83107b070a0ce47e0f5a67ab3c4bc597e246928b441a53da9dd0da","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_887d48f324f77d968eaecc1d","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2341eae9ac637f10786d03124a2b3f545948aa0c3bf3ac3aec3bb12da2dd8ff6","metric_details":{"license":"Proprietary","variance":2.308179394633107},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1451.0199915218975,"confidence_low":1445.0645650568129,"sample_count":76920,"score":1448.0422782893552,"score_display":"1448","source_stated_rank":82},"run_fingerprint":"1942c18ee737d673026eb7c7f5a827fcebfcc3140d6c324d733a9c973eb32f8f","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_aa1f16a5a40da24207af00f4","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"5c5ac354e3f7b6d2714677cc8cef6249e8c41df3ebad9a261cafec4da7700b8e","metric_details":{"license":"Proprietary","variance":3.112831913850149},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1453.238388859053,"confidence_low":1446.322373933448,"sample_count":49306,"score":1449.7803813962505,"score_display":"1450","source_stated_rank":80},"run_fingerprint":"e98f0be54c6c969710673647dd74bcc6c293ac1e863d9baead6df585105390f3","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f87012c55dbb863f49c465e0","provider_config":{"source_id":"arena-text","source_label":"thinking-16k","token_budget":16000},"provider_mode_key":"thinking-16k","raw_label":"thinking-16k","reasoning_enabled":null,"strategy":"token_budget","token_budget":16000},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"2e8e625db590ad520d3c64ecb97b640745bebf0ab5bf550e577f9a2c0597a02e","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.144678,"date_is_proxy":true,"latency_seconds":35.772,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.16},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":76.768,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":32000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-1-20250805-thinking","zero_shot_accuracy":75.758},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"a19313038bd94ef64a7e55ba771a5fa31a9d3b758a63545dfe0a8744c4a7bb65","result":{"confidence_high":80.4966,"confidence_low":72.02940000000001,"sample_count":396,"score":76.263,"score_display":"76.26","source_stated_rank":80},"run_fingerprint":"4b5cb9d152cf47d41fe53a0ea83c4b06dbb85d36b675021419a54ba5a5dd5ea1","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_99dd806d3a14424dd2a38e68","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8e961d9c4778d03145410fd41eb027b7461aa7e1009c512cf9fd7afeae3748a8","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.067421,"date_is_proxy":true,"latency_seconds":35.934,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.303},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":71.212,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":32000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":null,"scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-1-20250805","zero_shot_accuracy":68.687},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"abbd600330d40b33468fc367dd4fdec8ee8a1cb90acf8b3de0654e798827a50b","result":{"confidence_high":74.46388,"confidence_low":65.43612,"sample_count":396,"score":69.95,"score_display":"69.95","source_stated_rank":96},"run_fingerprint":"b549994dfc961e8c89ea9dd125c3117ec31010aae5932842205c2e656e24845e","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9ca9060f76a8c388c0751d7f","provider_config":{"source_id":"vals-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a79fbaffe036d5a7779fdb73f94431e28dd263955fd28e555df8c99d6350c877","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.124083,"date_is_proxy":true,"latency_seconds":27.668,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.325},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":32000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-1-20250805-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"dc427718882b3acc66d66e6cadeb66e71762b42dbdf2cb2465612fe379801d52","result":{"confidence_high":88.561,"confidence_low":87.287,"sample_count":null,"score":87.924,"score_display":"87.9","source_stated_rank":23},"run_fingerprint":"0e4d99c929ef2e6cfb5a1dccbde35a6bd848a9c8fd634971fd2249334eec5080","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_99dd806d3a14424dd2a38e68","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ac3329bcffa472fe602eff13cead68a947c40a29657fc25d0e5898e0b5fd718c","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.007078,"date_is_proxy":true,"latency_seconds":17.757,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.328},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":32000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-1-20250805"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"2bdfeec27c18668abdf6fcdb59836dcfb60c256d2109a95e7144ea16ac469d94","result":{"confidence_high":87.85688,"confidence_low":86.57112,"sample_count":null,"score":87.214,"score_display":"87.2","source_stated_rank":33},"run_fingerprint":"86cbf8b542fb91f842c4c47ab731d775fb2d2359b86c46249d6a04c999ec1510","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_165d3e02bdb077d596cd57fd","provider_config":{"source_id":"vals-mmlu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T21:15:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T21:15:07.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f547935789872195f8cd7800a9b72511a25c7ff42ba2fe2aa1f1e9c5f60a44ff","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.12631578947368421","leaderboard_private_problem_count":285,"model_release_date":"2025-08-05","public_example_count":10,"stderr":1.9712735463357798},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["32k thinking"],"epoch_id":"5eRkmXCURH2pRbhJoukfnC","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5eRkmXCURH2pRbhJoukfnC.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/5eRkmXCURH2pRbhJoukfnC.eval","manual_evaluation":"","reasoning_token_budget":32000,"service_tier":null,"source_model_display_name":"","thinking":{"budget_tokens":32000,"type":"token_budget"},"upstream_model_id":"claude-opus-4-1-20250805_32K","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"5b3536a86ff25f5a7597b213ff2cdf948aea0b0996d8daccf3472da9e0577a56","result":{"confidence_high":16.49527509818655,"confidence_low":8.767882796550293,"sample_count":285,"score":12.631578947368421,"score_display":"12.6","source_stated_rank":99},"run_fingerprint":"f769e1b09aa02d03ff35e07f08cde7b448b36ed23b6feaeda3faf34df6eb58b6","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_01129548b25adbb9776ec287","provider_config":{"source_id":"epoch-frontiermath","source_label":"thinking-32k","token_budget":32000},"provider_mode_key":"thinking-32k","raw_label":"thinking-32k","reasoning_enabled":null,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T21:15:07.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T21:15:07.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f70b51febfa4f2e8c80d8fc6db117a78ed00343ca4454b98151511d41b83d6dc","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.024390243902439025","leaderboard_private_problem_count":41,"model_release_date":"2025-08-05","public_example_count":2,"stderr":2.4390243902439024},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":["32k thinking"],"epoch_id":"9zT72xRR6wZhiwraJTNPyz","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9zT72xRR6wZhiwraJTNPyz.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/9zT72xRR6wZhiwraJTNPyz.eval","manual_evaluation":"","reasoning_token_budget":32000,"service_tier":null,"source_model_display_name":"","thinking":{"budget_tokens":32000,"type":"token_budget"},"upstream_model_id":"claude-opus-4-1-20250805_32K","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"ae135bcb39223edd49e0623f9c0ce7bc8c33cf3ee2ca71e01b0b431feced8c16","result":{"confidence_high":7.2195121951219505,"confidence_low":0.0,"sample_count":41,"score":2.4390243902439024,"score_display":"2.4","source_stated_rank":61},"run_fingerprint":"ef9263eb526bd97726cdcf82ee01d6d4ecfcbb1bab049e2267973bdac9ec051e","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_539047413da7106e5111873d","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"thinking-32k","token_budget":32000},"provider_mode_key":"thinking-32k","raw_label":"thinking-32k","reasoning_enabled":null,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"56cc8eb402bf0abd2982ae959b52235ce288c29a64e0f4e862fca2ff4fa7a024","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.168667,"date_is_proxy":true,"latency_seconds":2.446,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.004},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":32000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-1-20250805-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"f3d1889d8f364858d99f9b7530b5f532876dac5f2d0827f321ad6d7bdb83ee4f","result":{"confidence_high":79.48183999999999,"confidence_low":75.54616,"sample_count":null,"score":77.514,"score_display":"77.5","source_stated_rank":52},"run_fingerprint":"f9f7c9bfaf3f5ab9fd29ee714791d1c55e4baa07753a5216a667085c99c53b03","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_99dd806d3a14424dd2a38e68","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_c4828ed5eb385de228d39263","split_or_window":"vals-mmmu-pro-overall","unit":"percent","version":"Vals MMMU-Pro v1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"184a435fed2bd498da2642c68800a43b6e901f7ed28555f18f5e5f5825bdff7d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.044475,"date_is_proxy":true,"latency_seconds":46.277,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.058},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","max_output_tokens":32000,"protocol":"Vals AI independent run over MMMU-Pro; overall accuracy","reasoning":null,"reasoning_effort":null,"source_content_sha256":"ea435b1e8d317167a8a7620214923fb7c03f9d2338ed16176176462857fb53c2","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-1-20250805"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"da001bc9e1e87844751a0d9ebb33a9ad68c9260c8d990e374b86ebd18f803859","result":{"confidence_high":75.78868,"confidence_low":71.64132000000001,"sample_count":null,"score":73.715,"score_display":"73.7","source_stated_rank":58},"run_fingerprint":"6062b8d62f6952c76a19526997476352dcdb1cce7028a7124c9ba1daa98bdac3","source":{"content_hash":"272b8eecb5ccbf045cfcc747b3a38ace01bd209aac28cb69e0ef639b1fe3a7e6","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/mmmu","id":"vals-mmmu-pro","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMMU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmmu"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d1b552861ef5dc4b553d708c","provider_config":{"source_id":"vals-mmmu-pro","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"31dda79feea9fae6c4fc9a1dd99adfb470e9ab6e1cfbb94b06b570c16c4ab170","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-05","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-1-20250805","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"56504c70f5d45eef26f9b98e3fb6f9f02f5f3a2076da3de5c1556a7ca296352b","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.0,"score_display":"60.0","source_stated_rank":32},"run_fingerprint":"0102b3125201aa25834e4cef596c0408f198d34d127d2949a344331d8413e168","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dfc3eed7d38c3a0d0a68268a","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"473bfe93f8c5e6a5d0e164890e75e49d3cf55e37f896d6117191003aa2e07f39","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-05","notes":null,"upstream_source_url":"SimpleBench Leaderboard"},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-1-20250805_unknown","source_row_date":null,"upstream_leaderboard_url":""},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"db1da2354518061c9b065a0b2bba164ac3e00efe7de87b89237d684646c803f0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":60.0,"score_display":"60.0","source_stated_rank":32},"run_fingerprint":"d0d0dbc8e2f43ae824b225b193cfabefe7a44794abcb25dc864bd76acc648251","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_dfc3eed7d38c3a0d0a68268a","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":76,"at":"2026-07-20T18:43:50.000Z","basis":"evaluation_started_at","evaluated_at":"2026-07-20T18:43:50.000Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"fa611d33871c4c34ddbdaf0b26a169e8622fa18c452f44d595d3a36b77e6ae1f","metric_details":{"best_score_across_scorers":"0.07","model_release_date":"2025-08-05","stderr":2.564323999762428},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"cb9qw7m5zpPWKqRDLB4ktU","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fcb9qw7m5zpPWKqRDLB4ktU.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/cb9qw7m5zpPWKqRDLB4ktU.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"1c161d449f8413e0c84280f76f2c69fdb4f61212a2345d113816ad01603954ff","result":{"confidence_high":12.026075039534359,"confidence_low":1.9739249604656424,"sample_count":100,"score":7.000000000000001,"score_display":"7.0","source_stated_rank":148},"run_fingerprint":"0e11f5e35ebc9c9034c559c7d6c7d0bf84b0fe839ee36900982d876af5755a71","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_446c19809a2da3df55345cf2","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":235,"at":"2026-02-11T21:33:53.502Z","basis":"evaluation_started_at","evaluated_at":"2026-02-11T21:33:53.502Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"eb9348120b6b5399bc7f2916104f0a8ece71cf01f039a6619b6c6f8b94a90c3d","metric_details":{"best_score_across_scorers":"0.7334710743801653","model_release_date":"2025-08-05","stderr":2.0118245554677086},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"cA6Mq67yaynLhY8F5kes3c","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcA6Mq67yaynLhY8F5kes3c.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/cA6Mq67yaynLhY8F5kes3c.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"a2b04faf2230205e13b4c801da0beea2e8239f0cb8ddc3954f0007d9771dd68b","result":{"confidence_high":77.29028356673325,"confidence_low":69.40393130929982,"sample_count":484,"score":73.34710743801654,"score_display":"73.3","source_stated_rank":21},"run_fingerprint":"391888eea54099d97dece71c99690601bb5a1cda21cb09c7bbb6c0a3a29cd99b","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_832ee21d16c8e7eaa53c7f7c","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_e8fc75e5c8482265dbca3d4c","split_or_window":"2025-09-01/2025-12-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":308,"at":"2025-12-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c59ed59dc77980cb9cab597acd02da0e58240ba1e5d0a74df6415e4b36bc1a66","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-08-05","pass_at_5":54.421768707483,"potential_contamination":false,"sem":1.2909680970336588},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"Claude Opus 4.1__tools","window_from":"2025-09-01","window_status":"historical-post-release","window_to":"2025-12-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"122b54f28b6294368d79a290bafcab91ec40542b45f7262adfc41a652c854b80","result":{"confidence_high":45.99223018847878,"confidence_low":40.931635248106836,"sample_count":null,"score":43.46193271829281,"score_display":"43.46","source_stated_rank":null},"run_fingerprint":"0bcd0ff83cf8dbf795666b3b01cd87a0be2bc0433dd24521f2ecc20138ad87fb","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_4efca96ee4b3e901a2b47082","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"48cd42d4726235888b369b6144bf692a1b52519bc2eea8539aed15cca44f0fdc","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.360655,"date_is_proxy":true,"latency_seconds":92.167,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.125},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":32000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-1-20250805-thinking"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"9a619cbf9c7101c2f685542f06502bceab03a120d45c167a8964e2f422464278","result":{"confidence_high":68.661,"confidence_low":64.251,"sample_count":null,"score":66.456,"score_display":"66.5","source_stated_rank":99},"run_fingerprint":"b6ccc515902b559657cfbf528ad45fd9aa9eda5f27cdcc76930817026a502dfb","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"observed","effort_rank":null,"is_provider_default":false,"normalized_effort":"adaptive","profile_id":"profile_99dd806d3a14424dd2a38e68","provider_config":{"source_label":"adaptive"},"provider_mode_key":"thinking","raw_label":"adaptive","reasoning_enabled":true,"strategy":"adaptive","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"livecodebench","metric":"pass_at_1","name":"LiveCodeBench","release_id":"release_a2492ed53dd6d8c4d33d5ae5","split_or_window":"vals-livecodebench-overall","unit":"percent","version":"Vals LiveCodeBench v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"885699209db8deb78b08e20b5912cd749aa4d11a5c2980020553068dbe8e78c1","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.055988,"date_is_proxy":true,"latency_seconds":32.509,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.112},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","max_output_tokens":32000,"protocol":"Vals AI independent LiveCodeBench implementation; overall pass@1","reasoning":null,"reasoning_effort":null,"source_content_sha256":"288cbaaa1523d3e42a39ab6ad9ac5406ff83de0f6d22904e216990def0a0b0b7","temperature":1,"top_p":null,"upstream_model_id":"anthropic/claude-opus-4-1-20250805"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"code execution against hidden tests"},"protocol_fingerprint":"0527b0cfe1bde76f7e8aa1043f62f15a1bc90c690eb2030920864e858dbdba35","result":{"confidence_high":66.73852,"confidence_low":62.37948,"sample_count":null,"score":64.559,"score_display":"64.6","source_stated_rank":102},"run_fingerprint":"d9f92f506c8b3766c8917232cf0f9f92d1215d12b463f03224c2c347eea22a6d","source":{"content_hash":"ed856eccde62e24c3a4534647be49d943140d220755d0d26c6d88248638f1c91","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/lcb","id":"vals-livecodebench","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · LiveCodeBench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/lcb"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ada8ad6fe603edca0393fe74","provider_config":{"source_id":"vals-livecodebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"cd2fbb11d37c5eb5addb36439df4c2a43ae513a33482ec8216caba8407abbb49","metric_details":{"license":"Proprietary","variance":29.76890363831505},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"claude-opus-4-1-20250805","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"40686fb714e765972128c9724aa66e9c94792c455ca82f2bb1c24e9485ab0909","result":{"confidence_high":1400.128842192535,"confidence_low":1378.7413675904418,"sample_count":6926,"score":1389.4351048914884,"score_display":"1389","source_stated_rank":92},"run_fingerprint":"41ee87190894060816817d1f001bb14e6271ea58b5009a84a19a65aebe0f45a3","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0447edc48ed9dd787c5b6bc9","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-09-30T18:53:15Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"27c12674f5719f4b2cb66fce9ba52c479b2db2262374195b646c4f5227a9f76b","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-1-20250805_unknown","source_row_date":null,"upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"b69f867a8749b40785693373e90d0c0dd4c48cd5b1ca15944e98fcc2a932d5bf","result":{"confidence_high":1399.83,"confidence_low":1378.11,"sample_count":6769,"score":1388.97,"score_display":"1388.97","source_stated_rank":89},"run_fingerprint":"c6916c2024a1358fed5b5c646adbc65232c8d9b309b4b8dcc19d8b6666e5fad6","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a04abd5f35745260628bca53","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:56Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"a63c48e5ae80a01478ba6426dcb4e717f600973457a0f703c7193c28263e0695","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-05","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"claude-opus-4-1-20250805","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"2e063829f572c4da26df6278f1f57ce71d2ce6fa21869b07ffd8a2c7a05623d3","result":{"confidence_high":null,"confidence_low":null,"sample_count":8210,"score":1386.0,"score_display":"1386.00","source_stated_rank":92},"run_fingerprint":"26bb312f9e525858535121509473fe712fb073dfcb0fd425397d75c7b0caebe6","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a04abd5f35745260628bca53","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":335,"at":"2025-11-04","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-04","status":"stale"},"measurement_id":"58bebc5401926c1f13f6c3074ed6d0c0ca81d6a1a5710e653c7d1e8c2ed559ff","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-05","notes":null,"standard_error_points":2.9000000000000004,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-opus-4-1-20250805","source_row_date":"2025-11-04","source_scaffold":"Claude Code","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Claude Code","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"a64f3f73b1d1d890334ad3f5d9104b1e1c543c88983331688a6d2c9fdd7de3ba","result":{"confidence_high":40.483999999999995,"confidence_low":29.115999999999996,"sample_count":89,"score":34.8,"score_display":"34.8","source_stated_rank":123},"run_fingerprint":"1efb1d0080e9c3399428ed5019ea4ceeabe8736ed4c90dfa7d76d5d9431df48c","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_37c94b96d6b27897d2ae04cf","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":336,"at":"2025-11-03","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-03","status":"stale"},"measurement_id":"3b0121460021337b2c4939720dee2cdfa86845739a03724de6b1b7d9cc547af2","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-05","notes":null,"standard_error_points":2.5,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-opus-4-1-20250805","source_row_date":"2025-11-03","source_scaffold":"Mini-SWE-Agent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Mini-SWE-Agent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"e883e7083d9f2137a7fe93fc44378ecffbf37d0aab959ec7070eb0c0d25f3b70","result":{"confidence_high":39.99999999999999,"confidence_low":30.199999999999996,"sample_count":89,"score":35.099999999999994,"score_display":"35.1","source_stated_rank":119},"run_fingerprint":"dd2ecb3bb05178bc930282f8c645c8f46708cc4a1d0242fe7a9bc2a67780143f","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_37c94b96d6b27897d2ae04cf","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":337,"at":"2025-11-02","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-02","status":"stale"},"measurement_id":"cab04315be0a3f6bb1ba23539a56205d80cb24b27d5b8420f10a2f0c2b0887d3","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-05","notes":null,"standard_error_points":2.7,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-opus-4-1-20250805","source_row_date":"2025-11-02","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"e1b0ee89e0f450a5880da7d64296173af4f74e7d4717a1823ef11509aafc10cf","result":{"confidence_high":42.192,"confidence_low":31.607999999999997,"sample_count":89,"score":36.9,"score_display":"36.9","source_stated_rank":109},"run_fingerprint":"b98e2e82e5e70e7816c75f3407ed50a16a7e99455dfa74d46db05f319deb2ee6","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_37c94b96d6b27897d2ae04cf","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":339,"at":"2025-10-31","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-10-31","status":"stale"},"measurement_id":"fd608d5b9b9e71f47aad703db59d0721071c98d144270573d58567b1093dfc21","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-05","notes":null,"standard_error_points":2.6,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"claude-opus-4-1-20250805","source_row_date":"2025-10-31","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"865c852f39c5b8d13587231566138102d7f1cdc476b45efae9ceb6e17ca34581","result":{"confidence_high":43.096000000000004,"confidence_low":32.903999999999996,"sample_count":89,"score":38.0,"score_display":"38.0","source_stated_rank":107},"run_fingerprint":"deca39779c7a7058907997aef107c1208f49ccb36ee2e084884c92a5a382c536","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_37c94b96d6b27897d2ae04cf","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval","metric":"win_rate","name":"GDPval","release_id":"release_730527fa346470703bd19e21","split_or_window":"expert-comparison-win-rate","unit":"percent","version":"GDPval official leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:05Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:05Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"eb87132bc9f3bc68094f180a09d50ac5110929c1174b9fe76943967368ef8e62","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-08-05","notes":null,"upstream_source_url":null},"model":{"id":"anthropic/claude-opus-4-1","name":"Claude Opus 4.1","provider":"Anthropic","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Claude Opus 4.1","source_effort":null,"source_model_version":"claude-opus-4-1-20250805","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenAI GDPval expert-comparison protocol","tools_allowed":"as reported by the GDPval leaderboard"},"protocol_fingerprint":"2b9517ae4828c51845ede7bca6aadbc8eb35e299be4327218a6131dee37ed9fb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":43.6,"score_display":"43.6","source_stated_rank":3},"run_fingerprint":"5649dab3af1517ffc3c7fc8855dc6d44e684c0baa71a6643ee3acc53f24e6505","source":{"content_hash":"d4eac25e0863d96ee45cc545fc896e5667d7022da8b3bc3902533ed87fa8b836","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gdpval","id":"epoch-gdpval","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GDPval","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gdpval"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_73ca51c1241eb639e187e8eb","provider_config":{"source_id":"epoch-gdpval","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"410286578e9c16e7f12f40e67f7f7c5fbdc9f0983ace9ef372f530670abf6ac6","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2026-04-28"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":false,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Mistral Medium 3.5; model release: 2026-04-28","source_accessibility":"Open weights (restricted use)","source_model_name":"Mistral Medium 3.5","source_model_release_date":"2026-04-28","source_model_versions":["mistral-medium-2604"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"302139dd9971b3681179b779be322bed8fdcfdf8da4674b3584b450292e1253f","result":{"confidence_high":143.65,"confidence_low":138.38,"sample_count":null,"score":141.39,"score_display":"141.39","source_stated_rank":106},"run_fingerprint":"330ab021bebf5a75e8c9ea6bb6599728174ce17fb05cd2d87141e755f2cd9118","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_9df6cb8258aaa8e058f65ada","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"aa-intelligence-index","metric":"index_score","name":"Artificial Analysis Intelligence Index","release_id":"release_2e1f53ccb44d5bbb947dd199","split_or_window":"intelligence-index-v4-3","unit":"points","version":"Artificial Analysis Intelligence Index v4.3"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-09-30T18:53:23Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:23Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"5bdf72cb9e34e0c3130bf3af43bd3709cb7895a4d811908f8160a505ed45ce27","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-29","source_locator":"Leaderboard models table · Mistral Medium 3.5 · intelligenceIndex"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Weighted average of ten Artificial Analysis evaluations (Agents 30 %, Coding 20 %, Scientific Reasoning 20 %, General 30 %); September 2026 methodology; constituents: AA-Briefcase, GDPval-AA v2, AutomationBench-AA, Terminal-Bench v4.0, SciCode, AA-LCR v1.1, AA-Omniscience, HLE, GDP.pdf, CritPt","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mistral-medium-3-5","upstream_model_label":"Mistral Medium 3.5"},"run_count":null,"scaffold":null,"tools_allowed":"per evaluation"},"protocol_fingerprint":"d27e2cf2d09c672ec77d55692b08c3493d369efcb61d848b6bcf0ef173c140d3","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":14.188923,"score_display":"14.2","source_stated_rank":null},"run_fingerprint":"7e09b2f6e8c1f4d674c64f3d608d0a9db21bf0d48016891a5e1e7e6379402a54","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Mistral Medium 3.5 · intelligenceIndex","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_903b3bbc2f47321c04bfc588","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"c5e0c99baf60957d4376021dd8f2b2f9e1c001978dbada033570d3d4291f2cdf","metric_details":{"license":"Modified MIT","variance":10.700663534992275},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1433.3390256058044,"confidence_low":1420.516208704082,"sample_count":11757,"score":1426.9276171549432,"score_display":"1427","source_stated_rank":115},"run_fingerprint":"5ea01f4d83d7c4328eaf9e4cda92e1509c686c317394ea033b132d51a6d53c78","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_5ede15640eb2055da044aac7","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"expert-knowledge","higher_is_better":true,"id":"hle-text","metric":"accuracy","name":"Humanity's Last Exam · Text-only","release_id":"release_def1fd244db3612317ca0108","split_or_window":"text-only-2158-no-tools","unit":"percent","version":"HLE text-only (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T00:52:24Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T00:52:24Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"ec20130f6466c8e74b3f8d3adcf187e3e5647aa6ba9ac89fa309dc846ab73ff2","metric_details":{"comparison_warning":"Text-only HLE subset (2,158 of 2,500 questions), no tools; not directly comparable with the full multimodal lane.","comparison_warning_code":"hle-text-only-subset","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 2158 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-29","source_locator":"Leaderboard models table · Mistral Medium 3.5 · hle"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"hle-text-only-subset","judge":"GPT-5.6 Luna (medium) equality checker","run_config":{"date_is_proxy":true,"protocol":"Text-only subset (2,158 of 2,500 questions, May 2025 revision), no tools, pass@1, equality-checker grading","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mistral-medium-3-5","upstream_model_label":"Mistral Medium 3.5"},"run_count":1,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"fd01871c00b9f2c6cbc7f06d6458e58b51a749a18a48979049b33faa424bcc8b","result":{"confidence_high":15.280823,"confidence_low":12.373451,"sample_count":2158,"score":13.762743,"score_display":"13.8","source_stated_rank":null},"run_fingerprint":"6dc2421a4571617fb88b2bc450a7ab9458606fda35ef52969ffaeddf217ae8a5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Mistral Medium 3.5 · hle","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_903b3bbc2f47321c04bfc588","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_aa2f2a6181ec4d1bc71a32f8","split_or_window":"diamond-198-zero-and-five-shot-cot","unit":"percent","version":"GPQA Diamond v1 · Vals dual-prompt"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e144fcdaafcb5234dcffe08dc5122ed5b02d7c58dc23fbbb1b6d58a6d025e08d","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.095489,"date_is_proxy":true,"latency_seconds":256.17,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall dual-prompt accuracy","stderr":2.39},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"few_shot_accuracy":34.848,"leaderboard_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","max_output_tokens":80000,"prompt_variants":["diamond_zero_shot_cot","diamond_few_shot_cot"],"protocol":"198 GPQA Diamond questions evaluated once with zero-shot CoT and once with five-shot CoT; arithmetic mean of both accuracies","question_count":198,"reasoning":null,"reasoning_effort":"high","scored_responses":396,"source_content_sha256":"9ed92dc649509bd7147a35a5a702497aa1706d12d13556cfc8d056aea7423a3f","temperature":0.7,"top_p":0.95,"upstream_model_id":"mistralai/mistral-medium-3.5","zero_shot_accuracy":34.848},"run_count":null,"scaffold":"Vals AI fixed GPQA prompting and parser","tools_allowed":"none"},"protocol_fingerprint":"9242aae32eae8716e5b6fe3b294105085cd662a343b38dedf6549bb5b6a49fbf","result":{"confidence_high":39.532399999999996,"confidence_low":30.1636,"sample_count":396,"score":34.848,"score_display":"34.85","source_stated_rank":134},"run_fingerprint":"98c1ca430ceb74bc3d9b78dede1073c02d30d3a81ffe612c05c2430da1dbe5f9","source":{"content_hash":"3e936d9c93d852f585859a48502371eb07babfbcbbc899dd8cc84bec30ff36a1","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/gpqa","id":"vals-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall dual-prompt accuracy","name":"Vals AI GPQA Diamond","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/gpqa"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1e04e7bbc053f10a914aeca7","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_f83cfc2a9e19b8d654d7b58e","split_or_window":"diamond-198-zero-shot-5-repeats","unit":"percent","version":"GPQA Diamond (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"070959afbc8685b043d244e3988d993c6a65e7a80116cbeb3dbf8155e1fb3607","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 198 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-29","source_locator":"Leaderboard models table · Mistral Medium 3.5 · gpqa"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"198 Diamond questions, four-option multiple choice, regex extraction, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mistral-medium-3-5","upstream_model_label":"Mistral Medium 3.5"},"run_count":5,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"63d3c3930921846bfd6331edca570d49e31a5b1ae2c32042ff2d0cec876d646d","result":{"confidence_high":80.380033,"confidence_low":68.371067,"sample_count":198,"score":74.848485,"score_display":"74.8","source_stated_rank":null},"run_fingerprint":"7188f4c68aa1c02cee90dc6ab005fb99b35ca6a8a59a9296f67d3b4f2050cc47","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Mistral Medium 3.5 · gpqa","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_903b3bbc2f47321c04bfc588","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"knowledge","higher_is_better":true,"id":"mmlu-pro","metric":"accuracy","name":"MMLU-Pro","release_id":"release_187c8201a49a6385bb527d7d","split_or_window":"vals-mmlu-pro-overall","unit":"percent","version":"Vals MMLU-Pro v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:47Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"80d7eded1e69ef9476d1b11b6885d89a90b452a87d60f9d19988228bd6c7bb01","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.041763,"date_is_proxy":true,"latency_seconds":66.384,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.418},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","max_output_tokens":80000,"protocol":"Vals AI independent run over the MMLU-Pro question set; overall accuracy","reasoning":null,"reasoning_effort":"high","source_content_sha256":"16ad637b9dabfbe592e268f27ac9e9a991c635b5950bdfc4999ccbaec8c4844e","temperature":0.7,"top_p":0.95,"upstream_model_id":"mistralai/mistral-medium-3.5"},"run_count":null,"scaffold":"Vals AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"c71dd19e4e8ef1d8e37d58e354596a849bb9799d0044ae1f145e03f098610fd8","result":{"confidence_high":76.15428,"confidence_low":74.51571999999999,"sample_count":null,"score":75.335,"score_display":"75.3","source_stated_rank":111},"run_fingerprint":"70cc4a87b8727a48ad5f732bdfad4c9496a6689e79d0e9fa33a384d3964e4159","source":{"content_hash":"63a083ba5d86654e455c8f849aa2d58c38a44d9df5349cbb9f6054c85c001cb4","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-03T06:35:47Z","homepage_url":"https://www.vals.ai/benchmarks/mmlu_pro","id":"vals-mmlu-pro","last_fetched_at":"2026-10-05T12:33:33Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · MMLU-Pro","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/mmlu_pro"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1e04e7bbc053f10a914aeca7","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multimodal","higher_is_better":true,"id":"mmmu-pro","metric":"accuracy","name":"MMMU-Pro","release_id":"release_46396ea50bd4792fc96cf001","split_or_window":"aa-mmmu-pro","unit":"percent","version":"MMMU-Pro (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"a06b7594f7661abf278c972d7e0cfca78dad3715438a19d7f6e1d36fd12d5450","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-29","source_locator":"Leaderboard models table · Mistral Medium 3.5 · mmmuPro"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Artificial Analysis MMMU-Pro run; question subset and repeats as documented in the methodology","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mistral-medium-3-5","upstream_model_label":"Mistral Medium 3.5"},"run_count":null,"scaffold":null,"tools_allowed":"none"},"protocol_fingerprint":"f18a6fd4b876ff9bdd8bc71bb0ca167f04059d4da9ce03f1df4815e06d812dce","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":64.855491,"score_display":"64.9","source_stated_rank":null},"run_fingerprint":"9ecb2a52e5cdd0829ee096d94f0b8f6eaedde05758bfc9c48907fbf31033579f","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Mistral Medium 3.5 · mmmuPro","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_903b3bbc2f47321c04bfc588","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_a71cc2d98bd454276f8e6ed8","split_or_window":"70-challenges-official-grading","unit":"percent","version":"CritPt (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"547da72ff548a1406972f012e19154a53bfd23dad10d49f25cb23506560e236f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 70 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-29","source_locator":"Leaderboard models table · Mistral Medium 3.5 · critpt"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"70 test-set challenges graded by the official CritPt server, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mistral-medium-3-5","upstream_model_label":"Mistral Medium 3.5"},"run_count":5,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"2f0106ed78af3a27b4fcc8abfbb66bd81d562b865a016e1cc32af94c80f21a07","result":{"confidence_high":5.202487,"confidence_low":0.0,"sample_count":70,"score":0.0,"score_display":"0.0","source_stated_rank":null},"run_fingerprint":"584c5eecb22b088956db32938fbc776f320e132cd2b6747afac0b38df28daba1","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Mistral Medium 3.5 · critpt","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_903b3bbc2f47321c04bfc588","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a6c9e86c474dab229e628d3eb1de9ec1540a35d198d281ea996a5f01b37b797a","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":4.004623,"date_is_proxy":true,"latency_seconds":1143.68,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.114},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":80000,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":0.7,"top_p":0.95,"upstream_model_id":"mistralai/mistral-medium-3.5"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"5808db4e270d4dc0c16b04356e508982ed78afc450d4fcd518cfc3117a290944","result":{"confidence_high":70.54344,"confidence_low":62.25656000000001,"sample_count":500,"score":66.4,"score_display":"66.4","source_stated_rank":72},"run_fingerprint":"b8a6148f5c981dcf5e15e7310ccaae90001df42d43093db87cc5eee5296d25ea","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1e04e7bbc053f10a914aeca7","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_de40c66bd40a7b5483b40f9a","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode v1.0.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":29,"at":"2026-09-05T17:21:27Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-05T17:21:27Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9d13d4fe8d10a7fe3e14cbf6691f3c3cc2ce643246860d8118901d86108a0182","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 288 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-29","source_locator":"Leaderboard models table · Mistral Medium 3.5 · scicode"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"date_is_proxy":true,"protocol":"288 test-set subproblems with scientist-annotated background, pass@1 over 3 repeats; v1.0.1 regrading with more robust execution sandboxes","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mistral-medium-3-5","upstream_model_label":"Mistral Medium 3.5"},"run_count":3,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"d0dc3c00e75e5b65a54e2856921cf3e32e61833cee9fc208e0cba75faadd9901","result":{"confidence_high":45.917461,"confidence_low":34.665614,"sample_count":288,"score":40.162037,"score_display":"40.2","source_stated_rank":null},"run_fingerprint":"28222527c584fa79fc8e0c579bf4fcd4ff9cade6ed635245ad31826a55f9d62e","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Mistral Medium 3.5 · scicode","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_903b3bbc2f47321c04bfc588","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"1fff150f7e82e6b00c0d8a8a11e4f99d46f97b540cb7a1e81600af0ed6d39e8b","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":16.735503,"date_is_proxy":true,"latency_seconds":2470.533,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":1.713},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":80000,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":"high","source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":0.7,"top_p":0.95,"upstream_model_id":"mistralai/mistral-medium-3.5"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"a47165ec71deff5d76b843e5d6c5fb741af8c97ddec266118101110241f4017f","result":{"confidence_high":6.245480000000001,"confidence_low":0.0,"sample_count":null,"score":2.888,"score_display":"2.9","source_stated_rank":102},"run_fingerprint":"06455c57484a2134e4df7ac32bb830d7e70abb073b1e9824b04e36654c655590","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1e04e7bbc053f10a914aeca7","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a30ccb5bd4bb860649f48005f1f11b68cb68b48d2c57f7a289147765ed784eb5","metric_details":{"license":"Modified MIT","variance":56.662038311650704},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"mistral-medium-3.5","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"cfeb700379b466ae594387aece7b9cbf361b8a694faed86593090ddaad168897","result":{"confidence_high":1277.6817102208893,"confidence_low":1248.1747695546123,"sample_count":2322,"score":1262.9282398877508,"score_display":"1263","source_stated_rank":122},"run_fingerprint":"8693680e75a03c7fc08fbe85af5d381648d143aea2a8da408c5b326d6712803f","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_568ff51efc455e924f509855","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"accuracy","name":"Terminal-Bench 2.1","release_id":"release_3a1f88ce28d6c9608c34ad04","split_or_window":"vals-terminal-bench-2-1-overall","unit":"percent","version":"Vals Terminal-Bench 2.1 v2.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":7,"at":"2026-09-28","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:17Z","last_confirmed_at":"2026-10-03T04:34:58Z","observed_at":"2026-10-03T04:34:58Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"63ec960e895f7d3bd753ddb12adcb2799e8dc8ee99a8de55c819c3e9716e2061","metric_details":{"benchmark_page_updated":"2026-09-28","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.190984,"date_is_proxy":true,"latency_seconds":989.169,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.701},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","max_output_tokens":80000,"protocol":"Vals AI independent run over Terminal-Bench 2.1 with one uniform harness","reasoning":null,"reasoning_effort":"high","source_content_sha256":"53be5195c34da78e2b800eae5459e88ccb48880915297f78327cabdae4dedd00","temperature":0.7,"top_p":0.95,"upstream_model_id":"mistralai/mistral-medium-3.5"},"run_count":null,"scaffold":"Vals AI terminal agent harness","tools_allowed":"terminal and file tools in the Vals harness"},"protocol_fingerprint":"5c1e949b0587718d3317eba36fee09bdb91f1972089dadb92f0dd9dbbee6c530","result":{"confidence_high":44.24496,"confidence_low":33.65704,"sample_count":null,"score":38.951,"score_display":"39.0","source_stated_rank":70},"run_fingerprint":"e821d1ed5f19065d848378e1c615e73482b916c0a50af6c8a4e7d7306251370d","source":{"content_hash":"8eb0c27af7264580ae7d1571306bf2a8fb02c066fd6df1a6074be7b476ba419e","fetched_at":"2026-10-03T04:34:58Z","first_fetched_at":"2026-10-02T22:28:17Z","homepage_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1","id":"vals-terminal-bench-2-1","last_fetched_at":"2026-10-03T04:34:58Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Terminal-Bench 2.1","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/terminal-bench-2-1"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_1e04e7bbc053f10a914aeca7","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2.1","metric":"pass_at_1","name":"Terminal-Bench 2.1","release_id":"release_c53c830678318dc1763793c8","split_or_window":"terminus-2-89-tasks","unit":"percent","version":"Terminal-Bench 2.1 (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":33,"at":"2026-09-02T13:33:07Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T13:33:07Z","last_confirmed_at":"2026-09-15T17:15:32Z","observed_at":"2026-09-15T17:15:32Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"87da194924fdcc4a1383107b1e30b8f81a3f74415b7856a32b8d5a992abcbacb","metric_details":{"comparison_warning":"The source no longer lists this model on its current leaderboard. The value is kept unchanged from its last listing (last_confirmed_at) because the benchmark version and protocol are still the same; it is not a new measurement.","comparison_warning_code":"delisted-from-source-leaderboard","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 89 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-29","source_listing":{"last_listed_at":"2026-09-15T17:15:32Z","source_confirmed_at":"2026-10-02T19:14:24Z","status":"delisted-retained"},"source_locator":"Leaderboard models table · Mistral Medium 3.5 · terminalbenchV21"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window","source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":"delisted-from-source-leaderboard","judge":null,"run_config":{"date_is_proxy":true,"protocol":"Terminus 2 harness, 250 episodes and two-hour timeout per task, pass@1 over 3 repeats","reasoning_effort":"reported","source_content_sha256":"d1b1a5c0b5c02ea79700159959f61d17894690c81e258cbd1a7351232ae287a4","upstream_model_id":"mistral-medium-3-5","upstream_model_label":"Mistral Medium 3.5"},"run_count":3,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools in an E2B sandbox"},"protocol_fingerprint":"9011a703109755a222f872aded1d7159f707a2389a2810c58b110540d0f3f827","result":{"confidence_high":60.708728,"confidence_low":40.368376,"sample_count":89,"score":50.561798,"score_display":"50.6","source_stated_rank":null},"run_fingerprint":"1f0f3523e201bf23be9dd71428ee26ff2c286883fcca1735e15180bbbebd1c4e","source":{"content_hash":"3111b4d05ca7d005c83a7f0128df7261f4cc62ad9d2945e557c5e986b9dcaab0","fetched_at":"2026-09-15T17:15:32Z","first_fetched_at":"2026-09-15T17:15:32Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-09-15T17:15:32Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Mistral Medium 3.5 · terminalbenchV21","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_903b3bbc2f47321c04bfc588","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking-aa","metric":"pass_at_1","name":"τ³-Banking · Artificial Analysis harness","release_id":"release_786d72c2e6dda7b72acc4040","split_or_window":"banking-97-tasks-gpt-5-4-mini-simulator","unit":"percent","version":"τ³-Banking (Artificial Analysis)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":31,"at":"2026-09-04T09:03:13Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-04T09:03:13Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"d432ff2086b3de57648f99afedd90544bf3836e16da1e5677ba2e8611784d48f","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 97 scored items with a 95 % Wilson interval.","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-29","source_locator":"Leaderboard models table · Mistral Medium 3.5 · tauBanking"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"GPT-5.4 Mini (medium) user simulator and assertion judge","run_config":{"date_is_proxy":true,"protocol":"tau2-bench v1.0.1 dataset and grader, 200-step limit, pass@1 over 5 repeats","reasoning_effort":"reported","source_content_sha256":"b03ecb1b8232df4007efe23a6fabf6ecd6e533551b11a8b4b4c6dd8b0dfac048","upstream_model_id":"mistral-medium-3-5","upstream_model_label":"Mistral Medium 3.5"},"run_count":5,"scaffold":null,"tools_allowed":"tau2-bench tools with BM25/grep retrieval"},"protocol_fingerprint":"9c13619f06cb58301d59d7b5ab10aa6313710b3a0e1c1b6cdc0de17ef50bf44f","result":{"confidence_high":23.487957,"confidence_low":9.277885,"sample_count":97,"score":15.051546,"score_display":"15.1","source_stated_rank":null},"run_fingerprint":"6265c5ac4fbef5ce114b547589aad1ea3db614fb85ca74ec98bfe282f1088a54","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Mistral Medium 3.5 · tauBanking","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/leaderboards/models"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_903b3bbc2f47321c04bfc588","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"arena-agent","metric":"ips_score","name":"LMArena Agent Arena","release_id":"release_9a0495ba7c73bcdf294b57a2","split_or_window":"overall-ips-aggregate","unit":"ips","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"e43bacb68f2a176f31527ff9264b246cea2c7bc2a0cbcf5d9a79f21ca4a010b2","metric_details":{"confidence_level":0.95,"license":"Modified MIT","session_count":11156},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"source-specific","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","methodology":"inverse-propensity-scoring","source_model_id":"Mistral Medium 3.5","subset":"agent"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"f2235ab8cc502d46c3c61efbb122369342f87007f781af9c71f9f7c834b00b71","result":{"confidence_high":-0.10863917230960854,"confidence_low":-0.1390025386941619,"sample_count":437621,"score":-0.12382085550188522,"score_display":"-0.1238","source_stated_rank":49},"run_fingerprint":"ce3c7a1c7086fca682392217aeb5d0a11ac9d93f935a986c08c14bba74d72e22","source":{"content_hash":"cace136b85662e4e3c1f1d9ee926bcfe1864fcdc262482c7c323881eabe4d891","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/agent","id":"arena-agent","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena Agent Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/agent/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_92e35b6f8dac1e5fbe31fe5d","provider_config":{"source_id":"arena-agent","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"source-specific","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9a6b03a81981f10b40052cd97b6494e5d96a2b1befc164bdb30735a6477a3882","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":256000,"is_open_weights":true,"is_reasoning":true,"model_release_date":"2026-04-29","reported_confidence_interval":"-24 / +24","source_locator":"Leaderboard models table · Mistral Medium 3.5 · gdpval"},"model":{"id":"mistral/mistral-medium-3-5","name":"mistral Medium 3.5","provider":"Mistral AI","weights_status":"open"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"mistral-medium-3-5","upstream_model_label":"Mistral Medium 3.5"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"86abe9ca0f758f56dfb9f96f131726b8fd3b34e68e5795b49972dd7f1846e4c8","result":{"confidence_high":770.99,"confidence_low":723.21,"sample_count":220,"score":747.1,"score_display":"747","source_stated_rank":null},"run_fingerprint":"4e542712e0dcb87b91bcf1d000dc8695bbe58af2cc06cc1c8199fc9681be39fb","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Mistral Medium 3.5 · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_903b3bbc2f47321c04bfc588","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"4bdf06124b9e1f83abb9c62bd9a13d1e40a4e181de2e14eb637ce302c597e6bf","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-06-05"},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":true,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: Gemini 2.5 Pro (Jun 2025); model release: 2025-06-05","source_accessibility":"API access","source_model_name":"Gemini 2.5 Pro (Jun 2025)","source_model_release_date":"2025-06-05","source_model_versions":["gemini-2.5-pro","gemini-2.5-pro-preview-06-05","gemini-2.5-pro_16K","gemini-2.5-pro_32K"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"335d58d40f56d53147a2a0245492101b6455c53bc80de2cb53ef961e2aa9f700","result":{"confidence_high":146.87,"confidence_low":143.62,"sample_count":null,"score":145.31,"score_display":"145.31","source_stated_rank":81},"run_fingerprint":"d9ab4871e4b1a874af91999c265358c830f1843990c7659ce17c3c3420892e12","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_50cd9b2c1ec0a7e48d332239","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"bfafdd8ed7c8ca36e50c36dadd7507f131027278ef1b5d2d60b391d8224c138b","metric_details":{"license":"Proprietary","variance":1.5846901073734012},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1448.0335297382142,"confidence_low":1443.0989489896454,"sample_count":124834,"score":1445.5662393639298,"score_display":"1446","source_stated_rank":86},"run_fingerprint":"bdfc1caf8b45c31eeaa0efa7a14f63a06c4340e0dae2dafa9fdd94dc4a51fa12","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_2ce866b6b2c041b710946c67","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"gpqa-diamond","metric":"accuracy","name":"GPQA Diamond","release_id":"release_130743b4d612a7ac7c996be4","split_or_window":"diamond-epoch-inspect","unit":"percent","version":"Epoch independent Inspect runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":322,"at":"2025-11-16T19:00:08.124Z","basis":"evaluation_started_at","evaluated_at":"2025-11-16T19:00:08.124Z","first_observed_at":"2026-08-27T22:30:45Z","last_confirmed_at":"2026-10-05T12:33:33Z","observed_at":"2026-10-05T12:33:33Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f6a92ceabdd2cf104513709501ff7f63583521b463a73af02a31ffa6cdacab97","metric_details":{"best_score_across_scorers":"0.8529040404040404","model_release_date":"2025-06-17","stderr":2.117010842348584},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"cqUZ6gcq2HBnShFRWvKjGr","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcqUZ6gcq2HBnShFRWvKjGr.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/cqUZ6gcq2HBnShFRWvKjGr.eval","protocol":"Epoch AI independent evaluation; mean_score"},"run_count":null,"scaffold":"Epoch AI Inspect evaluation","tools_allowed":"none"},"protocol_fingerprint":"68a6e7cadceb43c5b9a9801b185bcaff2a90a6cca09be04d95158fc346e1666f","result":{"confidence_high":89.43974529140726,"confidence_low":81.14106278940082,"sample_count":null,"score":85.29040404040404,"score_display":"85.3","source_stated_rank":90},"run_fingerprint":"33675aab25b6dd6eba8bac3c35456e556bdc0e8ab254ef7917418aaa58e43fe3","source":{"content_hash":"30ffd73f640833a964453dd9a63934cd25e3e303fd52698df0dc351ee6d6fb06","fetched_at":"2026-10-05T12:33:33Z","first_fetched_at":"2026-10-02T02:26:10Z","homepage_url":"https://epoch.ai/benchmarks/gpqa-diamond","id":"epoch-gpqa-diamond","last_fetched_at":"2026-10-05T12:33:33Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI GPQA Diamond","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gpqa-diamond"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d8c9ceae61efafd815197e5f","provider_config":{"source_id":"epoch-gpqa-diamond","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"otis-mock-aime","metric":"mean_score","name":"OTIS Mock AIME 2024–2025","release_id":"release_071fe253e79f87573f6d22b0","split_or_window":"official-45-problem-set","unit":"percent","version":"2024–2025"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":322,"at":"2025-11-16T19:00:06.794Z","basis":"evaluation_started_at","evaluated_at":"2025-11-16T19:00:06.794Z","first_observed_at":"2026-08-28T19:57:30Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"c1238873b35494b0f35f51324fa20626b22053eb3b78a98c0f72909ea181829e","metric_details":{"best_score_across_scorers":"0.8472222222222222","model_release_date":"2025-06-17","stderr":4.814319605471135},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"iJp7PgYZa3PiXAV8UWd7MC","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FiJp7PgYZa3PiXAV8UWd7MC.eval","logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/iJp7PgYZa3PiXAV8UWd7MC.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"943eb42742aa7304c33a00a1b5ce982c6e59efcf39792521a9c166d82840c797","result":{"confidence_high":93.6027330933901,"confidence_low":74.73060023994324,"sample_count":45,"score":84.16666666666667,"score_display":"84.2","source_stated_rank":99},"run_fingerprint":"9e04e21d2d0e2d56fbee6230b44725f785acbd11d5af33ce3545bf6b07347faa","source":{"content_hash":"a366024baa9d0db55ae7ae86f29571e286ffbf2de1b447a015ca70b90c638f4e","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-01T20:29:17Z","homepage_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025","id":"epoch-otis-mock-aime","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · OTIS Mock AIME 2024–2025","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/otis-mock-aime-2024-2025"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8dd35dbcdf95c7644ed45a67","provider_config":{"source_id":"epoch-otis-mock-aime","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath","metric":"mean_score","name":"FrontierMath T1–3","release_id":"release_fd01c0ab187a88391c43ea9f","split_or_window":"private-285-of-295-problem-tiers-1-3-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T23:56:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T23:56:12.000Z","first_observed_at":"2026-08-31T21:58:01Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"7744ecaf1d0429f1d0ea5318da876e010aefbcfda2df893fae5b96a03850ddc3","metric_details":{"benchmark_total_problem_count":295,"best_score_across_scorers":"0.24561403508771928","leaderboard_private_problem_count":285,"model_release_date":"2025-06-17","public_example_count":10,"stderr":2.55425481026507},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"FkywQ2KS4AkLPzjSPmhHcS","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFkywQ2KS4AkLPzjSPmhHcS.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/FkywQ2KS4AkLPzjSPmhHcS.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"","thinking":null,"upstream_model_id":"gemini-2.5-pro","upstream_status":"Success","upstream_task_name":"FrontierMath-Tiers-1-3-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"3ee4a41c6fe93863e2309351ebad6a21d218136f9621743a0d47663fc12114c7","result":{"confidence_high":29.567742936891463,"confidence_low":19.555064080652393,"sample_count":285,"score":24.561403508771928,"score_display":"24.6","source_stated_rank":77},"run_fingerprint":"fb876a0c954b7e49d1c660dde772aa3ebb1a6c566a57e56bd02cf4b229494868","source":{"content_hash":"0560e9c4b7b88a6103284b75513a10c4ce5c82c6a25068563418319d0809090a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2","id":"epoch-frontiermath","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tiers 1–3 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tiers-1-3-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_1d3bc87d985da11182531266","provider_config":{"source_id":"epoch-frontiermath","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"ee9c089e8e0e09b15868572c5758e076014ef10e005763dcaf0ada54fbb61c87","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.757,"model_release_date":"2025-06-17T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-2-5-pro-2025-06-17-thinking","model_type":"CoT","upstream_model_id":"gemini-2-5-pro-2025-06-17-thinking-32k"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"e3324ed55ae00e7aa2305b98cdcb09439eabb0bb705055de1ec26d00a891530e","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4.859999999999999,"score_display":"5","source_stated_rank":165},"run_fingerprint":"e25f8a66b5c32363c707d8aa448f77530c724fc78ceaa6f22342596b00b408e6","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c837267eade2a8b2b7fc6bba","provider_config":{"source_id":"arc-agi-2","source_label":"Thinking 32K","token_budget":32000},"provider_mode_key":"Thinking 32K","raw_label":"Thinking 32K","reasoning_enabled":null,"strategy":"token_budget","token_budget":32000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"76ccd5bf2f93d0688427c952b723895be0303f8e9870c3dfe5bb1dd202267349","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.7145,"model_release_date":"2025-06-17T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-2-5-pro-2025-06-17-thinking","model_type":"CoT","upstream_model_id":"gemini-2-5-pro-2025-06-17-thinking-16k"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"207e1b4d3dd26508e354e68c454f4f14731eca5757aa83b6117647516bb9c75f","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":4.03,"score_display":"4","source_stated_rank":173},"run_fingerprint":"4ead2cbf1bfe28fca58d4a57c7c36be43d969f993bc2608faf600612a31bd327","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_291dc1adf261e0b67e48db67","provider_config":{"source_id":"arc-agi-2","source_label":"Thinking 16K","token_budget":16000},"provider_mode_key":"Thinking 16K","raw_label":"Thinking 16K","reasoning_enabled":null,"strategy":"token_budget","token_budget":16000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"74d7abcf2e24191f490688daf12b13c101d1c530d81f94cb906e38643c546fc6","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.4439,"model_release_date":"2025-06-17T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-2-5-pro-2025-06-17-thinking","model_type":"CoT","upstream_model_id":"gemini-2-5-pro-2025-06-17-thinking-8k"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"673de695b32acdcb9c62fc0287ccbae107bf3c77cad21c95562ce5c40e536e36","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":2.92,"score_display":"3","source_stated_rank":184},"run_fingerprint":"3a69e283cb8072995aa0706ccf0266d70ef2672f11ccb516fe3cf9c455a5f741","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ba4e557fea23dea51b9aefde","provider_config":{"source_id":"arc-agi-2","source_label":"Thinking 8K","token_budget":8000},"provider_mode_key":"Thinking 8K","raw_label":"Thinking 8K","reasoning_enabled":null,"strategy":"token_budget","token_budget":8000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"8e406494398d10ac165710799e9b7e36a4a082f7d9d8cafc4ade38e1afcfc722","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":0.0885,"model_release_date":"2025-06-17T00:00:00.000Z","results_url":""},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":"gemini-2-5-pro-2025-06-17-thinking","model_type":"CoT","upstream_model_id":"gemini-2-5-pro-2025-06-17-thinking-1k"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"bfe2fdece843f932599c16b967e11d61367159c6c871d4264165551aba5ff216","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.0,"score_display":"0","source_stated_rank":239},"run_fingerprint":"6cb5142f2b5a0007b5152b08e7c4596a445813c1586f7747e7a4b0ded8c0c196","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c01bd9581d08e2f8c5021e52","provider_config":{"source_id":"arc-agi-2","source_label":"Thinking 1K","token_budget":1000},"provider_mode_key":"Thinking 1K","raw_label":"Thinking 1K","reasoning_enabled":null,"strategy":"token_budget","token_budget":1000},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"mathematics","higher_is_better":true,"id":"frontiermath-tier-4","metric":"mean_score","name":"FrontierMath T4","release_id":"release_4d025b5eeb2f6b23fd7b19cb","split_or_window":"private-41-of-43-problem-tier-4-set","unit":"percent","version":"2.0.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":115,"at":"2026-06-11T23:56:12.000Z","basis":"evaluation_started_at","evaluated_at":"2026-06-11T23:56:12.000Z","first_observed_at":"2026-08-31T21:58:43Z","last_confirmed_at":"2026-10-05T12:33:22Z","observed_at":"2026-10-05T12:33:22Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"f9625d9607e29565a080be35826b73a50d82ea98d55eabc01ae9dfa9892a7a5b","metric_details":{"benchmark_total_problem_count":43,"best_score_across_scorers":"0.0","leaderboard_private_problem_count":41,"model_release_date":"2025-06-17","public_example_count":2,"stderr":0.0},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"display_qualifiers":[],"epoch_id":"2z7kL9DpfvY44T9Wo99FYx","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2z7kL9DpfvY44T9Wo99FYx.eval","logs":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2z7kL9DpfvY44T9Wo99FYx.eval","manual_evaluation":"","reasoning_token_budget":null,"service_tier":null,"source_model_display_name":"","thinking":null,"upstream_model_id":"gemini-2.5-pro","upstream_status":"Success","upstream_task_name":"FrontierMath-Tier-4-v2-Private","upstream_task_version":"2.0.0"},"run_count":null,"scaffold":"Epoch AI FrontierMath v2 Inspect evaluation harness","tools_allowed":"Python and submit_answer tools"},"protocol_fingerprint":"ea8444c769be78c97791bdc3eaecc535055ab1e12c18c8c196e36899b806dfce","result":{"confidence_high":0.0,"confidence_low":0.0,"sample_count":41,"score":0.0,"score_display":"0.0","source_stated_rank":66},"run_fingerprint":"9cef949f5849d531c9c0564a231653dad7ad44a7cc7cb08c45ed6720f431b3f9","source":{"content_hash":"e906462c488441e99c95015ee1dadcb88facb841efdaf236aa21b047c4f8077a","fetched_at":"2026-10-05T12:33:22Z","first_fetched_at":"2026-10-02T16:27:53Z","homepage_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2","id":"epoch-frontiermath-tier-4","last_fetched_at":"2026-10-05T12:33:22Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · FrontierMath Tier 4 v2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/frontiermath-tier-4-v2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_db12cac600249fd68f92b498","provider_config":{"source_id":"epoch-frontiermath-tier-4","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"science","higher_is_better":true,"id":"critpt","metric":"accuracy","name":"CritPt","release_id":"release_079aaa34e0885e15655a2847","split_or_window":"71-challenges-checkpoint-accuracy","unit":"percent","version":"CritPt current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:41Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"e9c0924cd29c223148ff4b9748dad0ff4f99e134a49d91b32ad31a467fe8b6ca","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 71 scored items with a 95 % Wilson interval.","cost":"85.05","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini-2.5 Pro","source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis CritPt evaluation","tools_allowed":"none reported"},"protocol_fingerprint":"df2751aa42beabc6279f00d6996d8d0a37d00a74060c6259906061f84e2a8176","result":{"confidence_high":8.480179,"confidence_low":0.447477,"sample_count":71,"score":2.0,"score_display":"2.0","source_stated_rank":132},"run_fingerprint":"97dd6fa4f02f2808b4daded713189b572e3c001a70b544e3df8785250a498cd2","source":{"content_hash":"11ee11feda8034d6db86ea13896e4339f839a412885eea280e52a53a28845991","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/critpt","id":"epoch-critpt","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · CritPt","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/critpt"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d0ea36065a785aad2b6b3bc6","provider_config":{"source_id":"epoch-critpt","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"games","higher_is_better":true,"id":"chess-puzzles","metric":"mean_score","name":"Chess Puzzles","release_id":"release_903e1366951b5b67ea61e8a3","split_or_window":"100-engine-generated-single-best-move-puzzles","unit":"percent","version":"Epoch AI Chess Puzzles v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":300,"at":"2025-12-08T14:58:40.497Z","basis":"evaluation_started_at","evaluated_at":"2025-12-08T14:58:40.497Z","first_observed_at":"2026-09-02T09:27:06Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"45cdc3fda3bc6308516e4f85f432d170332171f6ea6b898f5df5f22e6b7bc017","metric_details":{"best_score_across_scorers":"0.2","model_release_date":"2025-06-17","stderr":4.020151261036848},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"mQ3UwD6hUioWSp2s2oELY4","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmQ3UwD6hUioWSp2s2oELY4.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/mQ3UwD6hUioWSp2s2oELY4.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":"none"},"protocol_fingerprint":"fde96c687181c1fd3949e11375d0c6064a2123766aabdcd94114b94819521388","result":{"confidence_high":27.879496471632223,"confidence_low":12.120503528367777,"sample_count":100,"score":20.0,"score_display":"20.0","source_stated_rank":87},"run_fingerprint":"f74a846205e64362adb0d5b35568ebad160805b3d8c96865af70cc3c0a04bbe3","source":{"content_hash":"ccb94128e875219387cd59c5b2141c063ce38f76d8a877ceadb782090b47e3f1","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-01T20:29:23Z","homepage_url":"https://epoch.ai/benchmarks/chess-puzzles","id":"epoch-chess-puzzles","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI · Chess Puzzles","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/chess-puzzles"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e9ec569a89a44e56dca07dff","provider_config":{"source_id":"epoch-chess-puzzles","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_2242ce390e0c2cd53f1b3a5b","split_or_window":"verified-500-vals-mini-swe-agent","unit":"percent","version":"SWE-bench Verified v1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":34,"at":"2026-09-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T06:35:46Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"7c124b0dc36764b740c4611bf45df2e406a628ac210a110ac41aa913c5907dcb","metric_details":{"benchmark_page_updated":"2026-09-01","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":0.351578,"date_is_proxy":true,"latency_seconds":207.791,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":2.23},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","max_output_tokens":65536,"protocol":"Vals AI independent run over all 500 verified tasks; uniform minimal bash-only mini-swe-agent harness","reasoning":null,"reasoning_effort":null,"source_content_sha256":"7921082eb20942bc08123d70887ee210a4e6bfd6e5273e5967efad2354007c2a","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-2.5-pro"},"run_count":null,"scaffold":"Vals AI mini-swe-agent","tools_allowed":"Bash"},"protocol_fingerprint":"72a3f5561031f421278fbed0f98eed14bb93a36b49daf2ff63c36f2285689aae","result":{"confidence_high":58.7708,"confidence_low":50.029199999999996,"sample_count":500,"score":54.4,"score_display":"54.4","source_stated_rank":82},"run_fingerprint":"323309056b4946df643ab015c7547e4d16695030145e04eca1e7f3792b7204ac","source":{"content_hash":"59a4beea773aa12d64e90231e39a3bb9cc60c219bc263641c5fb35d8b2c61a3d","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-10-03T06:35:46Z","homepage_url":"https://www.vals.ai/benchmarks/swebench","id":"vals-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"Copyrighted leaderboard; minimal factual score fields with attribution","locator":"Results · Overall accuracy","name":"Vals AI SWE-bench Verified","redistribution_policy":"aggregate-facts-with-attribution-review","result_url":"https://www.vals.ai/benchmarks/swebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_7fbe7dfccaeef6bffdf6c1d5","provider_config":{"source_id":"vals-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-facts-with-attribution-review","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-bench-verified","metric":"resolved_rate","name":"SWE-bench Verified","release_id":"release_ab91f0254cf52d5b135599b2","split_or_window":"verified-484","unit":"percent","version":"Epoch independent runs"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":233,"at":"2026-02-13T18:40:15.413Z","basis":"evaluation_started_at","evaluated_at":"2026-02-13T18:40:15.413Z","first_observed_at":"2026-08-27T22:30:44Z","last_confirmed_at":"2026-10-05T12:33:32Z","observed_at":"2026-10-05T12:33:32Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"1277ebebf8c3cfbcc98859013a5d8847820516c5cf9c51efbda5a57ea96ed36a","metric_details":{"best_score_across_scorers":"0.5755693581780539","model_release_date":"2025-06-17","stderr":2.251275673889269},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-source","comparison_warning_code":null,"judge":null,"run_config":{"epoch_id":"KL724uHFGbiXj4o5PHyLn2","log_viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKL724uHFGbiXj4o5PHyLn2.eval","logs":"https://epoch-benchmarks-staging-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/KL724uHFGbiXj4o5PHyLn2.eval"},"run_count":null,"scaffold":"Epoch AI evaluation harness","tools_allowed":null},"protocol_fingerprint":"a08890121622db054f06ff540b7d74f65475f35ce7498fe9ce33b1a98dddc526","result":{"confidence_high":61.96943613862835,"confidence_low":53.144435496982425,"sample_count":484,"score":57.55693581780539,"score_display":"57.6","source_stated_rank":33},"run_fingerprint":"2c3fb7292e8076f68a3ff1c431b3f75339e0e8de9aed44e3b3279646ef0abf8f","source":{"content_hash":"871dbb4080905c0af1ef10b5fdc490e71663ef946e619c3c45e286e93f972419","fetched_at":"2026-10-05T12:33:32Z","first_fetched_at":"2026-09-02T21:28:30Z","homepage_url":"https://epoch.ai/benchmarks/swe-bench-verified","id":"epoch-swe-bench-verified","last_fetched_at":"2026-10-05T12:33:32Z","license":"CC-BY-4.0","locator":null,"name":"Epoch AI SWE-bench Verified","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/data/benchmark_data.zip"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_89d6ca3d37acbacedc5eda79","provider_config":{"source_id":"epoch-swe-bench-verified","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_started_at","date_rank":1,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"supplemental-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"swe-rebench-v2","metric":"resolved_rate","name":"SWE-rebench v2","release_id":"release_af7b312e6b2f98224b72c7a2","split_or_window":"2025-07-01/2025-11-01","unit":"percent","version":"rolling-v2"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":338,"at":"2025-11-01T00:00:00Z","basis":"benchmark_window_end","evaluated_at":null,"first_observed_at":"2026-08-30T11:02:42Z","last_confirmed_at":"2026-10-05T12:33:34Z","observed_at":"2026-10-05T12:33:34Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"610593c1596a7a95147c9a14684fa51c6be28aac8dcd312d993209adcf9d7285","metric_details":{"comparison_warning":"Historical official SWE-rebench snapshot; values from different rolling windows are not directly comparable.","comparison_warning_code":"swe-rebench-prior-window","model_release_date":"2025-06-17","pass_at_5":35.714285714285715,"potential_contamination":false,"sem":0.6343049053649439},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["noncanonical_source_or_window"],"is_primary":false},"protocol":{"comparison_lane":"supplemental-prior-window","comparison_warning_code":"swe-rebench-prior-window","judge":null,"run_config":{"agent_version":"tools","selection_rule":"widest-evidenced-post-release-window","upstream_model_id":"gemini-2.5-pro__tools","window_from":"2025-07-01","window_status":"historical-post-release","window_to":"2025-11-01"},"run_count":5,"scaffold":"SWE-rebench ReAct · tools","tools_allowed":"tools"},"protocol_fingerprint":"bb264d81b3cae436ede011e6918007fd00775d5dacdd100f8c9ca3f810d0ecab","result":{"confidence_high":24.703258256856376,"confidence_low":22.216783027825795,"sample_count":null,"score":23.460020642341085,"score_display":"23.46","source_stated_rank":null},"run_fingerprint":"09a036e3610988c04176a1e93c14010e3e2d8d6576a58a3dde386e98f2736187","source":{"content_hash":"eb56916eee2cf66075cd29f78d8ee8d3cf31054c43e3c5a0db95534e11c2ee1f","fetched_at":"2026-10-05T12:33:34Z","first_fetched_at":"2026-09-03T09:20:59Z","homepage_url":"https://swe-rebench.com/","id":"swe-rebench","last_fetched_at":"2026-10-05T12:33:34Z","license":"Leaderboard dataset CC-BY-4.0; attribution required","locator":null,"name":"SWE-rebench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://swe-rebench.com/"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c917d859ed1b346b2fb38d46","provider_config":{"source_id":"swe-rebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"benchmark_window_end","date_rank":2,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"supplemental-prior-window","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":0,"source_authority_tier":"A+","trust_rank":0,"trust_tier":"A+","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"gso","metric":"opt_at_1","name":"GSO","release_id":"release_176f07ea4e9e0b8dfeede990","split_or_window":"official-set-opt-at-1","unit":"percent","version":"GSO current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":360,"at":"2025-10-10","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:04Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-10-10","status":"stale"},"measurement_id":"3de2f7dc3822ed8d644e0a33e01f1288469bd8df3ac711a855c8e14783e2cd11","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"upstream_source_url":"https://gso-bench.github.io/index.html"},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":"2025-10-10","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools of the reported scaffold"},"protocol_fingerprint":"d1715ac84ac472c701ad292652c7db8141d54c54adb1365e0e996912f82d72c5","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":3.92,"score_display":"3.9","source_stated_rank":30},"run_fingerprint":"a20ac22fdac01437a6c6466c39f652a2ebc244dc33c2abaf79f2df354c8d82e0","source":{"content_hash":"b84875b3ce188a6d27484996a6f898ccc3b84b50cfe3576eeee52d71d020464a","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gso","id":"epoch-gso","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GSO","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gso"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cc9dc7454a55e2604ff083b8","provider_config":{"source_id":"epoch-gso","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"scicode","metric":"subproblem_accuracy","name":"SciCode","release_id":"release_6d948ae7a2a0a695cbdf21f9","split_or_window":"scientific-research-coding-subproblems","unit":"percent","version":"SciCode current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:47:54Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:47:54Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"c5ae536a8f6b96cccb709d58e49f684120639e099ad554dc28685f0f217254be","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"upstream_source_url":"https://artificialanalysis.ai/evaluations/scicode"},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 2.5 Pro","source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Artificial Analysis SciCode evaluation","tools_allowed":"Python execution against benchmark test cases"},"protocol_fingerprint":"592dc6f56ce994eb6fb20b61f31d50f6f9a6bc89908b88d344fae3c956567911","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":42.8240740740741,"score_display":"42.8","source_stated_rank":135},"run_fingerprint":"e65cdca29e98eebb1677a23ab641e3788584957039047e6f9592318df9466666","source":{"content_hash":"f5aee79ada1c9a958e233d176cfb319db8831c90edaf7a05666e46825c306d9b","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/scicode","id":"epoch-scicode","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SciCode","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/scicode"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_bba2d6a3b70be697c8ce9bfc","provider_config":{"source_id":"epoch-scicode","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding-agent","higher_is_better":true,"id":"vibe-code-bench","metric":"score","name":"Vibe Code Bench","release_id":"release_ee527953b92b7b6f6f1beb2d","split_or_window":"vals-vibe-code-bench-overall","unit":"percent","version":"Vals Vibe Code Bench v1.1"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-02T22:28:18Z","last_confirmed_at":"2026-10-03T04:34:59Z","observed_at":"2026-10-03T04:34:59Z","source_freshness":"stale","source_observation_age_hours":57.85,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"9ab30038877b030a84aa5a07e608a93b3ac3fd8170b5c67a0231c5f1c5f228ac","metric_details":{"benchmark_page_updated":"2026-10-02","comparison_warning":"Uniform Vals AI run. When no exact per-model publication date is exposed, the dated leaderboard version is shown only as a freshness proxy, not as the evaluation date; other harnesses remain separate.","comparison_warning_code":"vals-leaderboard-date-proxy","cost_per_test_usd":1.223831,"date_is_proxy":true,"latency_seconds":2097.144,"result_evidence_match":null,"result_evidence_url":null,"source_locator":"Results · Overall accuracy","stderr":0.4},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"vals-leaderboard-date-proxy","judge":null,"run_config":{"compute_effort":null,"leaderboard_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","max_output_tokens":65536,"protocol":"Vals AI independent run: building web applications from scratch; overall score","reasoning":null,"reasoning_effort":null,"source_content_sha256":"3057c4ecbc58ee2148be26e1d590f39235478955b609495ee48b22bd96426c8b","temperature":1,"top_p":null,"upstream_model_id":"google/gemini-2.5-pro"},"run_count":null,"scaffold":"OpenHands","tools_allowed":"coding-agent tools in the Vals harness"},"protocol_fingerprint":"0b42e01da1ca4630eeaf610722507fbe6a31af527b7db483e45132136514a1e9","result":{"confidence_high":1.1840000000000002,"confidence_low":0.0,"sample_count":null,"score":0.4,"score_display":"0.4","source_stated_rank":104},"run_fingerprint":"0874649661ade78f68cd4950a3d0ce26b8fc4648e0665ef8242e4b7d60c0c6c5","source":{"content_hash":"50f54310f86e32b50358edab9668866cbb0f6802e75c17d800799cf6ba358a25","fetched_at":"2026-10-03T04:34:59Z","first_fetched_at":"2026-10-02T22:28:18Z","homepage_url":"https://www.vals.ai/benchmarks/vibe-code","id":"vals-vibe-code-bench","last_fetched_at":"2026-10-03T04:34:59Z","license":"Copyrighted leaderboard; minimal factual score fields only","locator":"Results · Overall accuracy","name":"Vals AI · Vibe Code Bench","redistribution_policy":"minimal-factual-scores-with-attribution","result_url":"https://www.vals.ai/benchmarks/vibe-code"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_a1f1af34436462e553721abd","provider_config":{"source_id":"vals-vibe-code-bench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"minimal-factual-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"arena_score","name":"Text Arena · Coding","release_id":"release_c1b9d6012a41fe1d26bbd12e","split_or_window":"overall-code-arena","unit":"points","version":"2026-10-01"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":4,"at":"2026-10-01","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:29Z","observed_at":"2026-10-05T12:33:29Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"3769d19e1d9c5feb8b66f96a0b7fc4d506392375ec2f10e60d056c6752cdf85b","metric_details":{"license":"Proprietary","variance":65.09962091879983},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"category":"overall","dataset":"lmarena-ai/leaderboard-dataset","dataset_revision":"46919c467f7f93d9609b668a283bcd20d87c28bc","source_model_id":"gemini-2.5-pro","subset":"webdev"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"4c93204b5f1249e21b0081356862cadf34be7636a5bd1a19edcfffdd9ddccafb","result":{"confidence_high":1242.7926654800544,"confidence_low":1211.1649869055925,"sample_count":2729,"score":1226.9788261928234,"score_display":"1227","source_stated_rank":131},"run_fingerprint":"dd6d902f393160605fdc9cc04672117f10300fdc2a784097b333f49f827ff520","source":{"content_hash":"31f3808c55f655598548d7b1016c644a77234546d53c44c5d13d9328cbfced46","fetched_at":"2026-10-05T12:33:29Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/webdev","id":"arena-webdev","last_fetched_at":"2026-10-05T12:33:29Z","license":"CC-BY-4.0","locator":null,"name":"LMArena WebDev Arena","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/webdev/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cc37ec64586593de05640ffd","provider_config":{"source_id":"arena-webdev","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"webdev-arena","metric":"bradley_terry_rating","name":"Text Arena · Coding","release_id":"release_65fc140e7ffbb7bf62021f84","split_or_window":"text-arena-coding-bradley-terry","unit":"points","version":"Text Arena Coding current pool"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":273,"at":"2026-01-05","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-09-30T18:53:15Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-01-05","status":"stale"},"measurement_id":"f7c4c837be4acd2987c3f1322976858f1c9042c3c9145bfc73e3241282b5edbb","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"upstream_source_url":"https://web.lmarena.ai/leaderboard"},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":null,"source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":"2026-01-05","upstream_leaderboard_url":"https://web.lmarena.ai/leaderboard"},"run_count":null,"scaffold":"LMArena Text Arena coding web-app environment","tools_allowed":"React, TypeScript, Tailwind and declared dependencies in a sandbox"},"protocol_fingerprint":"ac00a042ca778107d8a3c56612faa4fc85d1e2d7903ecd9731d489f3645c71ef","result":{"confidence_high":1241.74,"confidence_low":1209.73,"sample_count":2667,"score":1225.73,"score_display":"1225.73","source_stated_rank":126},"run_fingerprint":"c6d12534579f668ce112a8edb3efdf0822a9a140a88821e68c5549ef366df3ea","source":{"content_hash":"d2cd62995fbfd609794475409f55ce7ab72996353a9ed43e9495215a19b54b9d","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/webdev-arena","id":"epoch-webdev-arena","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; LMArena leaderboard CC-BY-4.0","locator":null,"name":"Epoch AI mirror · Text Arena (Coding)","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/webdev-arena"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_36070ccd11ea356cb9ac51d4","provider_config":{"source_id":"epoch-webdev-arena","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":335,"at":"2025-11-04","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-04","status":"stale"},"measurement_id":"f677cfb31ff77ebee9304e95b1619adcf4190812e2dbdaefa80a0e98d4d56d5f","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"standard_error_points":2.9000000000000004,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":"2025-11-04","source_scaffold":"Gemini CLI","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Gemini CLI","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"8789675c2aba23fef0f10413d51bb9850a1b0435a2f3e78ffa4bf2327e2c6f94","result":{"confidence_high":25.284000000000002,"confidence_low":13.916,"sample_count":89,"score":19.6,"score_display":"19.6","source_stated_rank":170},"run_fingerprint":"4c97e029e8327ef19f6cfb5c9de7ae68f80a819f10a61726d6d27bf85104dec5","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c1e65ff3037c5f969ca962a0","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":336,"at":"2025-11-03","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-03","status":"stale"},"measurement_id":"944aba7ba25bdf513dd0bb10e09998aae7364b7a3d1d92559f1b84a704346835","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"standard_error_points":2.5,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":"2025-11-03","source_scaffold":"Mini-SWE-Agent","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Mini-SWE-Agent","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"6940e713db7f19d3e7b94d0d142abc89c1db0c0c4fb12fde7777a77643422daf","result":{"confidence_high":31.0,"confidence_low":21.200000000000003,"sample_count":89,"score":26.1,"score_display":"26.1","source_stated_rank":150},"run_fingerprint":"2ebab34f6a8b42d6a55dfc9a817e6d76c29ff262c27c3332b2c7b28b72ab50f1","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c1e65ff3037c5f969ca962a0","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":337,"at":"2025-11-02","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-11-02","status":"stale"},"measurement_id":"6af47c7bf6bb0ab1c333aa1497e83bcd132668cb91ae708686cee14eeff3b6e8","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"standard_error_points":2.8000000000000003,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":3,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":"2025-11-02","source_scaffold":"OpenHands","upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenHands","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"8c3d9e1a2d3d17e23135c2a60ee2191948d3cac835f739456d0c800bc909fd13","result":{"confidence_high":21.888,"confidence_low":10.912000000000003,"sample_count":89,"score":16.400000000000002,"score_display":"16.4","source_stated_rank":180},"run_fingerprint":"7e4c39fcc0d04485989f8a1cea356f64041c1d373b9e8ae10cdeb03b193216a1","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c1e65ff3037c5f969ca962a0","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"terminal-agent","higher_is_better":true,"id":"terminal-bench-2","metric":"accuracy_mean","name":"Terminal-Bench 2.0","release_id":"release_411f68def8d006fa2d493f44","split_or_window":"official-89-task-set","unit":"percent","version":"Terminal-Bench 2.0"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":339,"at":"2025-10-31","basis":"evaluation_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:55Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2025-10-31","status":"stale"},"measurement_id":"062e737d0ef18a7fb6e1e432ebdad6deb0a32c0e12596fdbc63ea796862a7423","metric_details":{"epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"standard_error_points":3.0,"upstream_source_url":"Terminal-Bench v2 Leaderboard"},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":4,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":false,"source_display_name":"","source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":"2025-10-31","source_scaffold":"Terminus 2","upstream_leaderboard_url":null},"run_count":null,"scaffold":"Terminus 2","tools_allowed":"terminal and file tools of the submitted agent harness"},"protocol_fingerprint":"93a1c20f2b8ca878f6f396e4857d3a6c290d7f6e2d3a40ecb5e62391b862285a","result":{"confidence_high":38.480000000000004,"confidence_low":26.720000000000002,"sample_count":89,"score":32.6,"score_display":"32.6","source_stated_rank":130},"run_fingerprint":"047c54c8f492c06b7d58c0ebaf09616e31f80d9901e091f64434355075904e76","source":{"content_hash":"2d378cab1a5b3352d78d26d23d3f8c5fd15d2081e3307d9c386bfb3f1f629179","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/terminal-bench","id":"epoch-terminal-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Terminal-Bench 2.0","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/terminal-bench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_c1e65ff3037c5f969ca962a0","provider_config":{"source_id":"epoch-terminal-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"tau3-banking","metric":"pass_1","name":"τ³-Banking · Official harness","release_id":"release_dc4a6b19eb552354004439bf","split_or_window":"banking_knowledge","unit":"percent","version":"1.0.1"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":153,"at":"2026-05-05","basis":"evaluation_at","evaluated_at":"2026-05-05","first_observed_at":"2026-08-28T00:52:14Z","last_confirmed_at":"2026-10-05T12:33:31Z","observed_at":"2026-10-05T12:33:31Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"stale"},"measurement_id":"ab98535a4f3a4183994930aed1ee668187db55cf05f5320333e7d47e881ecfef","metric_details":{"cost":null,"pass_2":6.01,"pass_3":2.58,"pass_4":1.03},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"domain":"banking_knowledge","retrieval_config":"alltools","submission_slug":"gemini-2-5-pro_sierra_2026-05-05","submission_type":"standard","user_simulator":"gpt-5.2"},"run_count":null,"scaffold":null,"tools_allowed":"tau3 standard tool set"},"protocol_fingerprint":"6246a2da066f68b80b217bb48423227a249c209670f7bdc5f29a22753e934992","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":13.66,"score_display":"13.7","source_stated_rank":null},"run_fingerprint":"6f275109bbb677df9ab05bff135a7c840263f4d5389e9cedbd585a3d81ee3c25","source":{"content_hash":"23400667e1eec0434194750dcfbbc96e62ebb9d8d023bc11d2b76508d0e0e1d6","fetched_at":"2026-10-05T12:33:31Z","first_fetched_at":"2026-09-30T18:53:20Z","homepage_url":"https://taubench.com/leaderboard/","id":"tau3-bench","last_fetched_at":"2026-10-05T12:33:31Z","license":"MIT","locator":null,"name":"τ³-Banking","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://raw.githubusercontent.com/sierra-research/tau2-bench/main/web/leaderboard/public/submissions/manifest.json"},"thinking":{"disclosure_status":"observed","effort_rank":4,"is_provider_default":false,"normalized_effort":"high","profile_id":"profile_331c7aa6d2358517713b57ec","provider_config":{"source_label":"high"},"provider_mode_key":"high","raw_label":"high","reasoning_enabled":true,"strategy":"fixed_effort","token_budget":null},"trust":{"date_basis":"evaluation_at","date_rank":0,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-submission","verified_status":"official-submission"}},{"benchmark":{"category":"tool-use","higher_is_better":true,"id":"apex-agents","metric":"pass_at_1","name":"APEX-Agents","release_id":"release_ca7e1389c4903b068f271e16","split_or_window":"480-professional-tasks-33-worlds","unit":"percent","version":"APEX-Agents current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:38:19Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:38:19Z","last_confirmed_at":"2026-09-21T22:33:38Z","observed_at":"2026-09-21T22:33:38Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-09-21","status":"fresh"},"measurement_id":"4245e5258190492682bc4ecc84d5c21a0b8c544dc5b68dc54f11e8d882ecee31","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-09-21","mean_score":"0.17","mean_standard_error":"1.9","model_release_date":"2025-06-17","notes":null,"source_listing":{"last_listed_at":"2026-09-21T22:33:38Z","source_confirmed_at":"2026-10-05T12:33:24Z","status":"delisted-retained"},"standard_error_points":1.5,"upstream_source_url":null},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 2.5 Pro (On)","source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Archipelago professional-work environment","tools_allowed":"documents, spreadsheets, presentations, PDF, email, chat, calendar, filesystem and code execution; no web search"},"protocol_fingerprint":"63d7ed2a91b8f00b11bbd9117e48d466e99c1fb1de9d7349e3b9f0015056906f","result":{"confidence_high":9.540000000000001,"confidence_low":3.6600000000000006,"sample_count":480,"score":6.6000000000000005,"score_display":"6.6","source_stated_rank":57},"run_fingerprint":"a767c4ddb9fdba930c60a5edeffc6ae482d3564df368fa3cd981752a79bc3126","source":{"content_hash":"de78d38ebc56b29bc20e427dc45815a829fccbb625a00bd2b8cfb2dd46872d3b","fetched_at":"2026-09-21T22:33:38Z","first_fetched_at":"2026-09-21T22:33:38Z","homepage_url":"https://epoch.ai/benchmarks/apex-agents","id":"epoch-apex-agents","last_fetched_at":"2026-09-21T22:33:38Z","license":"Epoch aggregate CC-BY-4.0; benchmark tasks and worlds CC-BY","locator":null,"name":"Epoch AI mirror · APEX-Agents","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/apex-agents"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_d50ec0394de26acc29a013ae","provider_config":{"source_id":"epoch-apex-agents","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"multi-turn-agent","higher_is_better":true,"id":"vending-bench-2","metric":"mean_final_balance_usd","name":"Vending-Bench 2","release_id":"release_143d5eb012c28bfa29646c4c","split_or_window":"one-simulated-year-mean-final-balance","unit":"USD","version":"Vending-Bench 2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:59Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:59Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"13589c5fc7846f6853c27bec79d3a0d43b056db8c8e226179068c4e8c746bff4","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 2.5 Pro","source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"Andon Labs vending-machine simulation","tools_allowed":"simulated business tools (email, ordering, pricing, inventory)"},"protocol_fingerprint":"08d84d8758b08cb2ccd6aba9c5097da0206a9dc1105085cc63b2a6423f43d9fd","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":573.6420000000003,"score_display":"573.64","source_stated_rank":52},"run_fingerprint":"48c4d5c59b5f3b6afe56a518c392530acc64f6a4c30d6f4a229d7262b2c6d2fc","source":{"content_hash":"43e5af166ea4d901f32bb54b62ec8f547ae86db0daa0e7d8649ddb145acbbae4","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/vending-bench-2","id":"epoch-vending-bench-2","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · Vending-Bench 2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/vending-bench-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ca43a0d026f7230ebd5478e2","provider_config":{"source_id":"epoch-vending-bench-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval-aa-v2.1","metric":"elo","name":"GDPval-AA v2.1","release_id":"release_2b38a48aaf3cb40aca8959b1","split_or_window":"220-tasks-elo-v2-1","unit":"points","version":"GDPval-AA v2.1 (Crowd-BT Elo, DeepSeek V4.1 Flash (max) at 1600)"},"conflict":null,"directly_comparable":false,"freshness":{"age_days":4,"at":"2026-10-01T09:31:57Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-10-01T09:31:57Z","last_confirmed_at":"2026-10-02T19:14:24Z","observed_at":"2026-10-02T19:14:24Z","source_freshness":"stale","source_observation_age_hours":67.19,"source_observation_status":"overdue","source_published_at":null,"status":"fresh"},"measurement_id":"f2e7402895c0b4d16a6c5ff13056e7f85472ed3f65956533d478f6f20130f39e","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","context_window_tokens":1000000,"is_open_weights":false,"is_reasoning":true,"model_release_date":"2025-06-05","reported_confidence_interval":"-25 / +25","source_locator":"Leaderboard models table · Gemini 2.5 Pro · gdpval"},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":false,"ineligibility_reasons":["source_observation_overdue"],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":"Pairwise panel of three frontier LLM judges (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash)","run_config":{"date_is_proxy":true,"protocol":"Pairwise Crowd-BT Elo anchored to DeepSeek V4.1 Flash (max) at 1600, 250 turns per task; GDPval-AA v2 tasks, sandbox and judge panel","reasoning_effort":"reported","source_content_sha256":"89630fa8ba0a602baeb21025a074b15e93be0048efc8115ec31e921a57b6a151","source_document_url":"https://artificialanalysis.ai/evaluations/gdpval-aa","upstream_model_id":"gemini-2-5-pro","upstream_model_label":"Gemini 2.5 Pro"},"run_count":1,"scaffold":"Stirrup (E2B sandbox)","tools_allowed":"web fetch, web search, view image, code execution"},"protocol_fingerprint":"94d70789a36ad4114b4e7b6e24afb961e62324c6c1a40a90540428a6f5e67885","result":{"confidence_high":null,"confidence_low":null,"sample_count":220,"score":443.74,"score_display":"444","source_stated_rank":null},"run_fingerprint":"12e3c1888fa5afdf27a533fe79cb9060ae545bedaeb953381157ea8d90b3a4e5","source":{"content_hash":"feee778b127c2b8d906036fb237b00c12191fae271133790ba417c3db9cb21b2","fetched_at":"2026-10-02T19:14:24Z","first_fetched_at":"2026-10-02T19:14:24Z","homepage_url":"https://artificialanalysis.ai/","id":"artificial-analysis","last_fetched_at":"2026-10-02T19:14:24Z","license":"Artificial Analysis leaderboard data; republication with attribution per agreement of 2026-09-02","locator":"Leaderboard models table · Gemini 2.5 Pro · gdpval","name":"Artificial Analysis · Intelligence Index evaluations","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://artificialanalysis.ai/evaluations/gdpval-aa"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_372da696bec592c74a30f371","provider_config":{"source_id":"artificial-analysis","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A","evidence_rank":1,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"independent-evaluation","verified_status":"independent-evaluation"}},{"benchmark":{"category":"professional-work","higher_is_better":true,"id":"gdpval","metric":"win_rate","name":"GDPval","release_id":"release_730527fa346470703bd19e21","split_or_window":"expert-comparison-win-rate","unit":"percent","version":"GDPval official leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:27:05Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:27:05Z","last_confirmed_at":"2026-10-05T12:33:27Z","observed_at":"2026-10-05T12:33:27Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"b0807324d9c29b2d1aea3770daf489558eea86e4a8488e6e89187309fef83ea8","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-06-17","notes":null,"upstream_source_url":null},"model":{"id":"google/gemini-2-5-pro","name":"Gemini 2.5 Pro","provider":"Google","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"Gemini 2.5 Pro","source_effort":null,"source_model_version":"gemini-2.5-pro","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"OpenAI GDPval expert-comparison protocol","tools_allowed":"as reported by the GDPval leaderboard"},"protocol_fingerprint":"4961034ca293169c4b6d01e6727df1eefe5de8d3761a099757d65461f173e1d8","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":23.3,"score_display":"23.3","source_stated_rank":9},"run_fingerprint":"9251e09bb796143595cfffdf98fe1bf4d23d6ee08c343aaecc2596914c544f9c","source":{"content_hash":"d4eac25e0863d96ee45cc545fc896e5667d7022da8b3bc3902533ed87fa8b836","fetched_at":"2026-10-05T12:33:27Z","first_fetched_at":"2026-10-05T12:33:27Z","homepage_url":"https://epoch.ai/benchmarks/gdpval","id":"epoch-gdpval","last_fetched_at":"2026-10-05T12:33:27Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · GDPval","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/gdpval"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_0b8a592e64ac30cd40ac2278","provider_config":{"source_id":"epoch-gdpval","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"capability-index","higher_is_better":true,"id":"epoch-eci","metric":"eci","name":"Epoch Capabilities Index","release_id":"release_bb97513e219727a9aa223cc4","split_or_window":"official-composite-fit","unit":"index_points","version":"Epoch ECI snapshot 2026-10-05"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":0,"at":"2026-10-05","basis":"dataset_version_published_at","evaluated_at":null,"first_observed_at":"2026-10-05T02:25:12Z","last_confirmed_at":"2026-10-05T12:33:21Z","observed_at":"2026-10-05T12:33:21Z","source_freshness":"current","source_observation_age_hours":1.88,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"3894caa753fc57b63eda74c7f9d59babc379227c7f5ed1cb1cf37a6abf9980f5","metric_details":{"confidence":[],"dataset_page_updated":"2026-10-05","model_release_date":"2025-02-27"},"model":{"id":"openai/gpt-4-5","name":"GPT 4.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{"aggregates_reasoning_profiles":false,"dataset_page":"https://epoch.ai/benchmarks/use-this-data","protocol":"Epoch ECI composite fit; upstream model: GPT-4.5; model release: 2025-02-27","source_accessibility":"API access","source_model_name":"GPT-4.5","source_model_release_date":"2025-02-27","source_model_versions":["gpt-4.5-preview-2025-02-27"],"source_reasoning_efforts":[]},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"6a4738e3b28616cc04c1b332379f4910d8724db672c7538a793122cabf8368ae","result":{"confidence_high":138.87,"confidence_low":133.81,"sample_count":null,"score":136.74,"score_display":"136.74","source_stated_rank":130},"run_fingerprint":"673c3194aa7cf2b57e770db321e26ff71e9562cd5a323e150f269bfe3eedda0d","source":{"content_hash":"0215c99486038474e3120978a57ef2e88ead9ad12ba82367a3cd566ec8bfc5b2","fetched_at":"2026-10-05T12:33:21Z","first_fetched_at":"2026-10-05T12:33:21Z","homepage_url":"https://epoch.ai/eci","id":"epoch-eci","last_fetched_at":"2026-10-05T12:33:21Z","license":"CC-BY-4.0","locator":null,"name":"Epoch Capabilities Index","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/eci"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_f9ca80d2335cd049580e7b95","provider_config":{"source_id":"epoch-eci","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"dataset_version_published_at","date_rank":5,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-benchmark-platform","verified_status":"official-benchmark-platform"}},{"benchmark":{"category":"preference","higher_is_better":true,"id":"arena-text","metric":"arena_score","name":"Arena Text","release_id":"release_47c553db7cf2b33a4a65c1f4","split_or_window":"overall-style-control","unit":"points","version":"2026-10-02"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":3,"at":"2026-10-02","basis":"leaderboard_published_at","evaluated_at":null,"first_observed_at":"2026-10-03T04:34:54Z","last_confirmed_at":"2026-10-05T12:33:28Z","observed_at":"2026-10-05T12:33:28Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"4d28603fbff28bbd133ec27fbee10573e50008e2c814a520399648abcbad2b36","metric_details":{"license":"Proprietary","variance":8.491674292249769},"model":{"id":"openai/gpt-4-5","name":"GPT 4.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":null,"judge":null,"run_config":{},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"0b93bb7cd821a0f9b8a64830b42202b12665b4817ae0e05104db2220c717e48f","result":{"confidence_high":1450.4026539651782,"confidence_low":1438.979796684721,"sample_count":14547,"score":1444.6912253249495,"score_display":"1445","source_stated_rank":87},"run_fingerprint":"f901499205502860cdd99f5706abf9b6abd353b2916b8caebbfe6035b3d40546","source":{"content_hash":"de40822d362ec8562ed40be9a3ee78075bb9835f8dcd2210422299b9b25ae1bb","fetched_at":"2026-10-05T12:33:28Z","first_fetched_at":"2026-10-03T04:34:54Z","homepage_url":"https://arena.ai/leaderboard/text","id":"arena-text","last_fetched_at":"2026-10-05T12:33:28Z","license":"CC-BY-4.0","locator":null,"name":"Arena Text","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/46919c467f7f93d9609b668a283bcd20d87c28bc/text_style_control/latest-00000-of-00001.parquet?download=true"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_8e9d4bb1423c318dd3622419","provider_config":{"source_id":"arena-text","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"leaderboard_published_at","date_rank":4,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-leaderboard","verified_status":"official-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"accuracy","name":"ARC-AGI-2","release_id":"release_1e2eafde2dbfd20577b0aed1","split_or_window":"semi-private-120","unit":"percent","version":"v2"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":38,"at":"2026-08-27T22:30:41Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-27T22:30:41Z","last_confirmed_at":"2026-10-05T12:33:30Z","observed_at":"2026-10-05T12:33:30Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":null,"status":"fresh"},"measurement_id":"a02cf4ac95dfa19c794fe40383f2041321c57458ff8829d7c7eca7a4ad313a1c","metric_details":{"comparison_warning":"The source publishes no date for this result. Freshness is based on the day this exact value was first observed by the collector (the value is at least that old); last_confirmed_at shows when the source last still listed it.","comparison_warning_code":"first-observed-date-proxy","cost_per_task_usd":2.1,"model_release_date":"2025-02-27T00:00:00.000Z","results_url":""},"model":{"id":"openai/gpt-4-5","name":"GPT 4.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"first-observed-date-proxy","judge":null,"run_config":{"dataset_id":"v2_Semi_Private","model_group":null,"model_type":"Base LLM","upstream_model_id":"gpt-4-5-2025-02-27"},"run_count":null,"scaffold":null,"tools_allowed":null},"protocol_fingerprint":"8ca700c75fd0c70f068a7efe5bd2209a9e8078fa9daf49318978ff9bc3376fb1","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":0.8,"score_display":"1","source_stated_rank":232},"run_fingerprint":"3e25c9b3869e50f810ddbae0ef7105538021eb72c4664f26b053213e444ad33f","source":{"content_hash":"8b1385616f496fb2c79a46cf9872781dd0fb32f63668e3ac20ec673ddfd264fc","fetched_at":"2026-10-05T12:33:30Z","first_fetched_at":"2026-10-01T20:29:26Z","homepage_url":"https://arcprize.org/leaderboard","id":"arc-agi-2","last_fetched_at":"2026-10-05T12:33:30Z","license":"Benchmark Apache-2.0; verified leaderboard scores redistributable with attribution (ARC Prize confirmation 2026-09-02)","locator":null,"name":"ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://arcprize.org/media/data/leaderboard/v2.json"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_e4904d86aa174e91395ae89f","provider_config":{"source_id":"arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"A+","evidence_rank":0,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":1,"source_authority_tier":"A","trust_rank":1,"trust_tier":"A","verification_status":"official-verified-leaderboard","verified_status":"official-verified-leaderboard"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-2","metric":"pass_at_2_accuracy","name":"ARC-AGI-2","release_id":"release_c66f5b82bd3045b91ad807bc","split_or_window":"evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":37,"at":"2026-08-28T22:43:25Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-08-28T22:43:25Z","last_confirmed_at":"2026-10-05T12:33:24Z","observed_at":"2026-10-05T12:33:24Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"773b3e73e744dde420464ffaba1e914512e8cabe70f524797c4e2eabbe511e95","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","confidence_interval_basis":"derived-wilson-95","confidence_interval_note":"Not published by the source. Derived from the score and the 360 scored items with a 95 % Wilson interval.","cost_per_task":"2.1","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-02-27","notes":null,"upstream_source_url":null},"model":{"id":"openai/gpt-4-5","name":"GPT 4.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":2,"eligible":true,"ineligibility_reasons":[],"is_primary":false},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-4.5","source_effort":null,"source_model_version":"gpt-4.5-preview-2025-02-27","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-2 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"734aee732feee9571883106c568c0ce2896a6dc1a25ff63d0c1e73e5ddd38173","result":{"confidence_high":2.371983,"confidence_low":0.266968,"sample_count":360,"score":0.8,"score_display":"0.8","source_stated_rank":223},"run_fingerprint":"53601b639462b4d153c88dbaa9d4c6988012a7c2bc72adcf5e11a96afb889a5d","source":{"content_hash":"07b6e47e8da8fd4e2ca9cea0052b1cce34f06b9c1212d90ff7bfe10fb8ade5c9","fetched_at":"2026-10-05T12:33:24Z","first_fetched_at":"2026-10-05T12:33:24Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi-2","id":"epoch-arc-agi-2","last_fetched_at":"2026-10-05T12:33:24Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-2","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi-2"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_cb9f607f31395822b15998d0","provider_config":{"source_id":"epoch-arc-agi-2","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"reasoning","higher_is_better":true,"id":"simplebench","metric":"accuracy_avg_at_5","name":"SimpleBench","release_id":"release_af5e4653e36272fb4c37bce2","split_or_window":"official-set-avg-at-5","unit":"percent","version":"SimpleBench current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:45Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:45Z","last_confirmed_at":"2026-10-05T12:33:25Z","observed_at":"2026-10-05T12:33:25Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"202f3a5aa80cb6e5ec8bc4452aae8eb6f8d8d399f3462433f30c0b43e3a08dcd","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-02-27","notes":null,"upstream_source_url":"https://simple-bench.com/"},"model":{"id":"openai/gpt-4-5","name":"GPT 4.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-4.5-preview-2025-02-27","source_row_date":null,"upstream_leaderboard_url":"https://simple-bench.com/"},"run_count":5,"scaffold":"SimpleBench official evaluation","tools_allowed":"none"},"protocol_fingerprint":"37aa265f7e46135b03120441fe4e99ea0e0e63e53b2f1e5f0246e366ff5aaadb","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":34.5,"score_display":"34.5","source_stated_rank":78},"run_fingerprint":"6e78582271745c2203a63e10f1691b91f3340e00c3f3de0b0447aed413018af3","source":{"content_hash":"18bc92bdd3fb847b80243bf3ab5188024fec46fda6fc44b09743978a1e12ed08","fetched_at":"2026-10-05T12:33:25Z","first_fetched_at":"2026-10-05T12:33:25Z","homepage_url":"https://epoch.ai/benchmarks/simplebench","id":"epoch-simplebench","last_fetched_at":"2026-10-05T12:33:25Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · SimpleBench","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/simplebench"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_ef0bc472e1202ec8035b0373","provider_config":{"source_id":"epoch-simplebench","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"abstraction","higher_is_better":true,"id":"arc-agi-1","metric":"pass_at_2_accuracy","name":"ARC-AGI-1","release_id":"release_2e65e63d3852e53d984c2949","split_or_window":"semi-private-evaluation-pass-at-2","unit":"percent","version":"ARC-AGI-1 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:47Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:47Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"256192ce022a2688cefa40b9ebb100ebc5a3efb7008bee0f246cc9f024f430aa","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","cost_per_task":"0.29","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-02-27","notes":null,"upstream_source_url":"ARC Prize Leaderboard"},"model":{"id":"openai/gpt-4-5","name":"GPT 4.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":"GPT-4.5","source_effort":null,"source_model_version":"gpt-4.5-preview-2025-02-27","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"reported ARC-AGI-1 evaluation configuration","tools_allowed":"none beyond the submitted reasoning system"},"protocol_fingerprint":"bd224672784620385214a71c3882741fc7c3d00550de99d815118f5c98564380","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":10.299999999999999,"score_display":"10.3","source_stated_rank":250},"run_fingerprint":"5ae523552f09d9d07780d924abc3ccca428cb8734b289e80bb828d13b4e58efc","source":{"content_hash":"33d6822d2f3b375f0bf8ac7fcaedc56826dd15424acb0ad21f169d95e68c46fb","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/arc-agi","id":"epoch-arc-agi-1","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · ARC-AGI-1","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/arc-agi"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_fc7c5235ad5211414647a803","provider_config":{"source_id":"epoch-arc-agi-1","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}},{"benchmark":{"category":"coding","higher_is_better":true,"id":"weirdml","metric":"accuracy","name":"WeirdML","release_id":"release_ed6368cba447631077bb553f","split_or_window":"weirdml-v2-task-set","unit":"percent","version":"WeirdML v2 current leaderboard"},"conflict":null,"directly_comparable":true,"freshness":{"age_days":33,"at":"2026-09-02T09:26:51Z","basis":"first_observed_at","evaluated_at":null,"first_observed_at":"2026-09-02T09:26:51Z","last_confirmed_at":"2026-10-05T12:33:26Z","observed_at":"2026-10-05T12:33:26Z","source_freshness":"current","source_observation_age_hours":1.87,"source_observation_status":"current","source_published_at":"2026-10-05","status":"fresh"},"measurement_id":"964da3a99fff2215dad0e1cefded7ef5a5746c526a0c18d1b7a2930cfacc2095","metric_details":{"comparison_warning":"Epoch mirrors this benchmark's official leaderboard, but the source does not publish a per-run evaluation date. The dated Epoch dataset version is used only as a freshness proxy.","comparison_warning_code":"epoch-external-dataset-date-proxy","epoch_dataset_updated":"2026-10-05","model_release_date":"2025-02-27","notes":null,"upstream_source_url":"https://htihle.github.io/weirdml.html"},"model":{"id":"openai/gpt-4-5","name":"GPT 4.5","provider":"OpenAI","weights_status":"closed"},"primary":{"display_priority":1,"eligible":true,"ineligibility_reasons":[],"is_primary":true},"protocol":{"comparison_lane":"canonical-source","comparison_warning_code":"epoch-external-dataset-date-proxy","judge":null,"run_config":{"dataset_page":"https://epoch.ai/benchmarks/use-this-data","date_is_proxy":true,"source_display_name":null,"source_effort":null,"source_model_version":"gpt-4.5-preview-2025-02-27","source_row_date":null,"upstream_leaderboard_url":null},"run_count":null,"scaffold":"WeirdML code-execution harness","tools_allowed":"Python execution with iterative feedback"},"protocol_fingerprint":"de1924d7fdc9c73ed108eae58e1410a4f66ff18dd72b55411187ca678704e6c0","result":{"confidence_high":null,"confidence_low":null,"sample_count":null,"score":39.37,"score_display":"39.4","source_stated_rank":116},"run_fingerprint":"7d946c140c1502d0fb5a7855ff918cde3ba0e3d25bf6e78391a7381069fe092c","source":{"content_hash":"ddfc71c9fbd054cbef617f973ca26614c4ef51c40d295dcf1c0c6e1ed5dfb375","fetched_at":"2026-10-05T12:33:26Z","first_fetched_at":"2026-10-05T12:33:26Z","homepage_url":"https://epoch.ai/benchmarks/weirdml","id":"epoch-weirdml","last_fetched_at":"2026-10-05T12:33:26Z","license":"Epoch aggregate CC-BY-4.0; upstream terms retained","locator":null,"name":"Epoch AI mirror · WeirdML","redistribution_policy":"aggregate-scores-with-attribution","result_url":"https://epoch.ai/benchmarks/weirdml"},"thinking":{"disclosure_status":"undisclosed","effort_rank":null,"is_provider_default":false,"normalized_effort":"unknown","profile_id":"profile_802f9a30c260e60eda4c12f6","provider_config":{"source_id":"epoch-weirdml","source_label":"reported"},"provider_mode_key":null,"raw_label":"reported","reasoning_enabled":null,"strategy":"unknown","token_budget":null},"trust":{"date_basis":"first_observed_at","date_rank":6,"evidence_grade":"B","evidence_rank":3,"protocol_lane":"canonical-source","public_export_eligible":true,"redistribution_policy":"aggregate-scores-with-attribution","source_authority_rank":2,"source_authority_tier":"B","trust_rank":2,"trust_tier":"B","verification_status":"official-secondary-verification","verified_status":"official-secondary-verification"}}],"meta":{"api_version":"1.11.0","attribution":{"note":"Upstream benchmark owners are credited per measurement in source_name, source_url and source_license; reuse of a single value carries that source's attribution as well.","required":true,"text":"Source: LLM Benchmarks (https://llmbenchmarks.io/)","url":"https://llmbenchmarks.io/"},"canonical_url":"https://llmbenchmarks.io/api/v1/measurements.json","dataset_schema_version":"1.13.0","generated_at":"2026-10-05T14:25:51Z","license":"Database compilation: CC-BY-4.0; upstream licenses apply per result.","missing_value":"null means not evaluated or not available for the exact configuration","snapshot_fingerprint":"208f757b9879aedec4187c35db8d263ed8b1d072c5dac72178687558d4d931f6"}}